From bb6d04be3b6e9320cb162280fed843ee6d2d12c1 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 8 Sep 2023 14:54:02 +0300 Subject: [PATCH 001/438] Logic for handling rule chain updates from edge, including metadata --- .../service/edge/rpc/EdgeGrpcSession.java | 14 +- .../service/edge/rpc/EdgeSyncCursor.java | 2 +- .../edge/rpc/processor/BaseEdgeProcessor.java | 3 + .../rule/BaseRuleChainProcessor.java | 127 ++++++++++++++++++ .../rule/RuleChainEdgeProcessor.java | 102 +++++++++++++- .../server/edge/AbstractEdgeTest.java | 46 ++----- .../server/edge/AssetEdgeTest.java | 26 ++++ .../server/edge/DashboardEdgeTest.java | 23 ++++ .../server/edge/DeviceEdgeTest.java | 31 +++++ .../server/edge/RuleChainEdgeTest.java | 72 +++++----- .../server/dao/rule/RuleChainService.java | 2 + common/edge-api/src/main/proto/edge.proto | 5 +- .../server/dao/rule/BaseRuleChainService.java | 15 ++- 13 files changed, 390 insertions(+), 78 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index cfb92d0bfd..538a12bd08 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -65,6 +65,8 @@ import org.thingsboard.server.gen.edge.v1.RequestMsg; import org.thingsboard.server.gen.edge.v1.RequestMsgType; import org.thingsboard.server.gen.edge.v1.ResponseMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; @@ -628,7 +630,7 @@ public final class EdgeGrpcSession implements Closeable { case CUSTOMER: return ctx.getCustomerProcessor().convertCustomerEventToDownlink(edgeEvent); case RULE_CHAIN: - return ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent); + return ctx.getRuleChainProcessor().convertRuleChainEventToDownlink(edgeEvent, this.edgeVersion); case RULE_CHAIN_METADATA: return ctx.getRuleChainProcessor().convertRuleChainMetadataEventToDownlink(edgeEvent, this.edgeVersion); case ALARM: @@ -690,6 +692,16 @@ public final class EdgeGrpcSession implements Closeable { result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); } } + if (uplinkMsg.getRuleChainUpdateMsgCount() > 0) { + for (RuleChainUpdateMsg ruleChainUpdateMsg : uplinkMsg.getRuleChainUpdateMsgList()) { + result.add(ctx.getRuleChainProcessor().processRuleChainMsgFromEdge(edge.getTenantId(), edge, ruleChainUpdateMsg)); + } + } + if (uplinkMsg.getRuleChainMetadataUpdateMsgCount() > 0) { + for (RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg : uplinkMsg.getRuleChainMetadataUpdateMsgList()) { + result.add(ctx.getRuleChainProcessor().processRuleChainMetadataMsgFromEdge(edge.getTenantId(), edge, ruleChainMetadataUpdateMsg)); + } + } if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { result.add(ctx.getAlarmProcessor().processAlarmMsg(edge.getTenantId(), alarmUpdateMsg)); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index a39dae3b54..586e1b403a 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -53,7 +53,6 @@ public class EdgeSyncCursor { public EdgeSyncCursor(EdgeContextComponent ctx, Edge edge, boolean fullSync) { if (fullSync) { fetchers.add(new QueuesEdgeEventFetcher(ctx.getQueueService())); - fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService())); fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService(), ctx.getFreemarkerConfig())); fetchers.add(new TenantEdgeEventFetcher(ctx.getTenantService())); fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService())); @@ -64,6 +63,7 @@ public class EdgeSyncCursor { fetchers.add(new CustomerUsersEdgeEventFetcher(ctx.getUserService(), edge.getCustomerId())); } } + fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService())); fetchers.add(new DashboardsEdgeEventFetcher(ctx.getDashboardService())); fetchers.add(new DefaultProfilesEdgeEventFetcher(ctx.getDeviceProfileService(), ctx.getAssetProfileService())); fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService())); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 328a64efe2..78bd56d90c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -233,6 +233,9 @@ public abstract class BaseEdgeProcessor { @Autowired protected DataValidator entityViewValidator; + @Autowired + protected DataValidator ruleChainValidator; + @Autowired protected EdgeMsgConstructor edgeMsgConstructor; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java new file mode 100644 index 0000000000..dc9fa6e14d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java @@ -0,0 +1,127 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.rule; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.NodeConnectionInfo; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; +import org.thingsboard.server.gen.edge.v1.RuleNodeProto; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Function; + +@Slf4j +public class BaseRuleChainProcessor extends BaseEdgeProcessor { + + protected boolean saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg) { + boolean created = false; + RuleChain ruleChain = ruleChainService.findRuleChainById(tenantId, ruleChainId); + if (ruleChain == null) { + created = true; + ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setCreatedTime(Uuids.unixTimestamp(ruleChainId.getId())); + } + ruleChain.setName(ruleChainUpdateMsg.getName()); + ruleChain.setType(RuleChainType.EDGE); + ruleChain.setDebugMode(ruleChainUpdateMsg.getDebugMode()); + ruleChain.setConfiguration(JacksonUtil.toJsonNode(ruleChainUpdateMsg.getConfiguration())); + + UUID firstRuleNodeUUID = safeGetUUID(ruleChainUpdateMsg.getFirstRuleNodeIdMSB(), ruleChainUpdateMsg.getFirstRuleNodeIdLSB()); + ruleChain.setFirstRuleNodeId(firstRuleNodeUUID != null ? new RuleNodeId(firstRuleNodeUUID) : null); + + ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); + if (created) { + ruleChain.setId(ruleChainId); + } + ruleChainService.saveRuleChain(ruleChain, false); + return created; + } + + protected boolean saveOrUpdateRuleChainMetadata(TenantId tenantId, RuleChainId ruleChainId, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) throws IOException { + RuleChainMetaData ruleChainMetadata = new RuleChainMetaData(); + ruleChainMetadata.setRuleChainId(ruleChainId); + ruleChainMetadata.setNodes(parseNodeProtos(ruleChainId, ruleChainMetadataUpdateMsg.getNodesList())); + ruleChainMetadata.setConnections(parseConnectionProtos(ruleChainMetadataUpdateMsg.getConnectionsList())); + ruleChainMetadata.setRuleChainConnections(parseRuleChainConnectionProtos(ruleChainMetadataUpdateMsg.getRuleChainConnectionsList())); + if (ruleChainMetadataUpdateMsg.getFirstNodeIndex() != -1) { + ruleChainMetadata.setFirstNodeIndex(ruleChainMetadataUpdateMsg.getFirstNodeIndex()); + } + if (ruleChainMetadata.getNodes().size() > 0) { + ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetadata, Function.identity()); + return true; + } + return false; + } + + private List parseNodeProtos(RuleChainId ruleChainId, List nodesList) throws IOException { + List result = new ArrayList<>(); + for (RuleNodeProto proto : nodesList) { + RuleNode ruleNode = new RuleNode(); + RuleNodeId ruleNodeId = new RuleNodeId(new UUID(proto.getIdMSB(), proto.getIdLSB())); + ruleNode.setId(ruleNodeId); + ruleNode.setCreatedTime(Uuids.unixTimestamp(ruleNodeId.getId())); + ruleNode.setRuleChainId(ruleChainId); + ruleNode.setType(proto.getType()); + ruleNode.setName(proto.getName()); + ruleNode.setDebugMode(proto.getDebugMode()); + ruleNode.setConfiguration(JacksonUtil.OBJECT_MAPPER.readTree(proto.getConfiguration())); + ruleNode.setAdditionalInfo(JacksonUtil.OBJECT_MAPPER.readTree(proto.getAdditionalInfo())); + result.add(ruleNode); + } + return result; + } + + private List parseConnectionProtos(List connectionsList) { + List result = new ArrayList<>(); + for (org.thingsboard.server.gen.edge.v1.NodeConnectionInfoProto proto : connectionsList) { + NodeConnectionInfo info = new NodeConnectionInfo(); + info.setFromIndex(proto.getFromIndex()); + info.setToIndex(proto.getToIndex()); + info.setType(proto.getType()); + result.add(info); + } + return result; + } + + private List parseRuleChainConnectionProtos(List ruleChainConnectionsList) throws IOException { + List result = new ArrayList<>(); + for (org.thingsboard.server.gen.edge.v1.RuleChainConnectionInfoProto proto : ruleChainConnectionsList) { + RuleChainConnectionInfo info = new RuleChainConnectionInfo(); + info.setFromIndex(proto.getFromIndex()); + info.setTargetRuleChainId(new RuleChainId(new UUID(proto.getTargetRuleChainIdMSB(), proto.getTargetRuleChainIdLSB()))); + info.setType(proto.getType()); + info.setAdditionalInfo(JacksonUtil.OBJECT_MAPPER.readTree(proto.getAdditionalInfo())); + result.add(info); + } + return result; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index ba6fc1ef97..ae5e158ba7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -15,29 +15,120 @@ */ package org.thingsboard.server.service.edge.rpc.processor.rule; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; import static org.thingsboard.server.service.edge.DefaultEdgeNotificationService.EDGE_IS_ROOT_BODY_KEY; @Component @Slf4j @TbCoreComponent -public class RuleChainEdgeProcessor extends BaseEdgeProcessor { +public class RuleChainEdgeProcessor extends BaseRuleChainProcessor { + + public ListenableFuture processRuleChainMsgFromEdge(TenantId tenantId, Edge edge, RuleChainUpdateMsg ruleChainUpdateMsg) { + log.trace("[{}] executing processRuleChainMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainUpdateMsg, edge.getName()); + RuleChainId ruleChainId = new RuleChainId(new UUID(ruleChainUpdateMsg.getIdMSB(), ruleChainUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (ruleChainUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + RuleChain ruleChainToDelete = ruleChainService.findRuleChainById(tenantId, ruleChainId); + if (ruleChainToDelete != null) { + ruleChainService.unassignRuleChainFromEdge(tenantId, ruleChainId, edge.getId(), false); + } + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(ruleChainUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + if (e.getMessage().contains("limit reached")) { + log.warn("[{}] Number of allowed rule chains violated {}", tenantId, ruleChainUpdateMsg, e); + return Futures.immediateFuture(null); + } else { + return Futures.immediateFailedFuture(e); + } + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + + public ListenableFuture processRuleChainMetadataMsgFromEdge(TenantId tenantId, Edge edge, RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg) { + log.trace("[{}] executing processRuleChainMetadataMsgFromEdge [{}] from edge [{}]", tenantId, ruleChainMetadataUpdateMsg, edge.getName()); + RuleChainId ruleChainId = new RuleChainId(new UUID(ruleChainMetadataUpdateMsg.getRuleChainIdMSB(), ruleChainMetadataUpdateMsg.getRuleChainIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (ruleChainMetadataUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateRuleChainMetadata(tenantId, ruleChainId, ruleChainMetadataUpdateMsg); + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(ruleChainMetadataUpdateMsg.getMsgType()); + } + } catch (Exception e) { + String errMsg = String.format("Can't process rule chain metadata update msg %s", ruleChainMetadataUpdateMsg); + log.error(errMsg, e); + return Futures.immediateFailedFuture(new RuntimeException(errMsg, e)); + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, Edge edge) { + boolean created = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), ruleChainId); + pushRuleChainCreatedEventToRuleEngine(tenantId, edge, ruleChainId); + ruleChainService.assignRuleChainToEdge(tenantId, ruleChainId, edge.getId()); + } + if (ruleChainUpdateMsg.getRoot()) { + edge.setRootRuleChainId(ruleChainId); + edgeService.saveEdge(edge); + } + } + + private void pushRuleChainCreatedEventToRuleEngine(TenantId tenantId, Edge edge, RuleChainId ruleChainId) { + try { + RuleChain ruleChain = ruleChainService.findRuleChainById(tenantId, ruleChainId); + String ruleChainAsString = JacksonUtil.toString(ruleChain); + TbMsgMetaData msgMetaData = getEdgeActionTbMsgMetaData(edge, null); + pushEntityEventToRuleEngine(tenantId, ruleChainId, null, TbMsgType.ENTITY_CREATED, ruleChainAsString, msgMetaData); + } catch (Exception e) { + log.warn("[{}][{}] Failed to push rule chain action to rule engine: {}", tenantId, ruleChainId, TbMsgType.ENTITY_CREATED.name(), e); + } + } - public DownlinkMsg convertRuleChainEventToDownlink(EdgeEvent edgeEvent) { + public DownlinkMsg convertRuleChainEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { RuleChainId ruleChainId = new RuleChainId(edgeEvent.getEntityId()); DownlinkMsg downlinkMsg = null; switch (edgeEvent.getAction()) { @@ -55,10 +146,13 @@ public class RuleChainEdgeProcessor extends BaseEdgeProcessor { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); RuleChainUpdateMsg ruleChainUpdateMsg = ruleChainMsgConstructor.constructRuleChainUpdatedMsg(msgType, ruleChain, isRoot); + RuleChainMetaData ruleChainMetaData = ruleChainService.loadRuleChainMetaData(edgeEvent.getTenantId(), ruleChainId); + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = + ruleChainMsgConstructor.constructRuleChainMetadataUpdatedMsg(edgeEvent.getTenantId(), msgType, ruleChainMetaData, edgeVersion); downlinkMsg = DownlinkMsg.newBuilder() .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) .addRuleChainUpdateMsg(ruleChainUpdateMsg) - .build(); + .addRuleChainMetadataUpdateMsg(ruleChainMetadataUpdateMsg).build(); } break; case DELETED: diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index 410348762f..dc1015ef4a 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -81,14 +81,12 @@ import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.QueueUpdateMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.SyncCompletedMsg; import org.thingsboard.server.gen.edge.v1.TenantProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.TenantUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; -import org.thingsboard.server.gen.edge.v1.UplinkMsg; import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; import org.thingsboard.server.queue.util.DataDecodingEncodingService; @@ -129,36 +127,12 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { installation(); edgeImitator = new EdgeImitator("localhost", 7070, edge.getRoutingKey(), edge.getSecret()); - edgeImitator.expectMessageAmount(26); + edgeImitator.expectMessageAmount(27); edgeImitator.connect(); - requestEdgeRuleChainMetadata(); - verifyEdgeConnectionAndInitialData(); } - private void requestEdgeRuleChainMetadata() throws Exception { - RuleChainId rootRuleChainId = getEdgeRootRuleChainId(); - RuleChainMetadataRequestMsg.Builder builder = RuleChainMetadataRequestMsg.newBuilder() - .setRuleChainIdMSB(rootRuleChainId.getId().getMostSignificantBits()) - .setRuleChainIdLSB(rootRuleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(builder); - UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder() - .addRuleChainMetadataRequestMsg(builder.build()); - edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); - } - - private RuleChainId getEdgeRootRuleChainId() throws Exception { - List edgeRuleChains = doGetTypedWithPageLink("/api/edge/" + edge.getUuidId() + "/ruleChains?", - new TypeReference>() {}, new PageLink(100)).getData(); - for (RuleChain edgeRuleChain : edgeRuleChains) { - if (edgeRuleChain.isRoot()) { - return edgeRuleChain.getId(); - } - } - throw new RuntimeException("Root rule chain not found"); - } - @After public void teardownEdgeTest() { try { @@ -230,7 +204,7 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { // 2 messages - 1 from rule chain fetcher and 1 from rule chain controller UUID ruleChainUUID = validateRuleChains(); - // 1 from request message + // 2 messages - 1 from rule chain fetcher and 1 from rule chain controller (it goes along with RuleChainUpdateMsg) validateRuleChainMetadataUpdates(ruleChainUUID); // 4 messages - 4 messages from fetcher - 2 from system level ('mail', 'mailTemplates') and 2 from admin level ('mail', 'mailTemplates') @@ -385,11 +359,17 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { } private void validateRuleChainMetadataUpdates(UUID expectedRuleChainUUID) { - Optional ruleChainMetadataUpdateOpt = edgeImitator.findMessageByType(RuleChainMetadataUpdateMsg.class); - Assert.assertTrue(ruleChainMetadataUpdateOpt.isPresent()); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = ruleChainMetadataUpdateOpt.get(); - Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsg.getMsgType()); - UUID ruleChainUUID = new UUID(ruleChainMetadataUpdateMsg.getRuleChainIdMSB(), ruleChainMetadataUpdateMsg.getRuleChainIdLSB()); + List ruleChainMetadataUpdateMsgList = edgeImitator.findAllMessagesByType(RuleChainMetadataUpdateMsg.class); + Assert.assertEquals(2, ruleChainMetadataUpdateMsgList.size()); + // metadata create msg + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsgCreated = ruleChainMetadataUpdateMsgList.get(0); + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, ruleChainMetadataUpdateMsgCreated.getMsgType()); + UUID ruleChainUUID = new UUID(ruleChainMetadataUpdateMsgCreated.getRuleChainIdMSB(), ruleChainMetadataUpdateMsgCreated.getRuleChainIdLSB()); + Assert.assertEquals(expectedRuleChainUUID, ruleChainUUID); + // metadata update msg + RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsgUpdated = ruleChainMetadataUpdateMsgList.get(1); + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, ruleChainMetadataUpdateMsgUpdated.getMsgType()); + ruleChainUUID = new UUID(ruleChainMetadataUpdateMsgUpdated.getRuleChainIdMSB(), ruleChainMetadataUpdateMsgUpdated.getRuleChainIdLSB()); Assert.assertEquals(expectedRuleChainUUID, ruleChainUUID); } diff --git a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java index faf0896b81..e3ad8aaa25 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java @@ -168,6 +168,7 @@ public class AssetEdgeTest extends AbstractEdgeTest { public void testSendAssetToCloud() throws Exception { UUID uuid = Uuids.timeBased(); + // created asset on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); assetUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); @@ -191,6 +192,31 @@ public class AssetEdgeTest extends AbstractEdgeTest { Asset asset = doGet("/api/asset/" + uuid, Asset.class); Assert.assertNotNull(asset); Assert.assertEquals("Asset Edge 2", asset.getName()); + + // updated asset on edge + uplinkMsgBuilder = UplinkMsg.newBuilder(); + assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); + assetUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + assetUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + assetUpdateMsgBuilder.setName("Asset Edge 2 Updated"); + assetUpdateMsgBuilder.setType("test"); + assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder); + uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + asset = doGet("/api/asset/" + uuid, Asset.class); + Assert.assertNotNull(asset); + Assert.assertEquals("Asset Edge 2 Updated", asset.getName()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index 2d21d30174..7ef3da1fa0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -171,6 +171,7 @@ public class DashboardEdgeTest extends AbstractEdgeTest { public void testSendDashboardToCloud() throws Exception { UUID uuid = Uuids.timeBased(); + // create dashboard on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); dashboardUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); @@ -191,6 +192,28 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Dashboard dashboard = doGet("/api/dashboard/" + uuid, Dashboard.class); Assert.assertNotNull(dashboard); Assert.assertEquals("Edge Test Dashboard", dashboard.getName()); + + // update dashboard on edge + uplinkMsgBuilder = UplinkMsg.newBuilder(); + dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); + dashboardUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + dashboardUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + dashboardUpdateMsgBuilder.setTitle("Edge Test Dashboard Updated"); + dashboardUpdateMsgBuilder.setConfiguration(""); + dashboardUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(dashboardUpdateMsgBuilder); + uplinkMsgBuilder.addDashboardUpdateMsg(dashboardUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + dashboard = doGet("/api/dashboard/" + uuid, Dashboard.class); + Assert.assertNotNull(dashboard); + Assert.assertEquals("Edge Test Dashboard Updated", dashboard.getName()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java index 15b15c0be0..42f8d1d272 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceEdgeTest.java @@ -538,6 +538,7 @@ public class DeviceEdgeTest extends AbstractEdgeTest { public void testSendDeviceToCloud() throws Exception { UUID uuid = Uuids.timeBased(); + // create device on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); DeviceUpdateMsg.Builder deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder(); deviceUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); @@ -566,6 +567,36 @@ public class DeviceEdgeTest extends AbstractEdgeTest { Device device = doGet("/api/device/" + newDeviceId, Device.class); Assert.assertNotNull(device); Assert.assertEquals("Edge Device 2", device.getName()); + + // update device on edge + uplinkMsgBuilder = UplinkMsg.newBuilder(); + deviceUpdateMsgBuilder = DeviceUpdateMsg.newBuilder(); + deviceUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + deviceUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + deviceUpdateMsgBuilder.setName("Edge Device 2 Updated"); + deviceUpdateMsgBuilder.setType("test"); + deviceUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + uplinkMsgBuilder.addDeviceUpdateMsg(deviceUpdateMsgBuilder.build()); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.expectMessageAmount(1); + + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + Assert.assertTrue(edgeImitator.waitForMessages()); + + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof DeviceCredentialsRequestMsg); + latestDeviceCredentialsRequestMsg = (DeviceCredentialsRequestMsg) latestMessage; + Assert.assertEquals(uuid.getMostSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdMSB()); + Assert.assertEquals(uuid.getLeastSignificantBits(), latestDeviceCredentialsRequestMsg.getDeviceIdLSB()); + + newDeviceId = new UUID(latestDeviceCredentialsRequestMsg.getDeviceIdMSB(), latestDeviceCredentialsRequestMsg.getDeviceIdLSB()); + + device = doGet("/api/device/" + newDeviceId, Device.class); + Assert.assertNotNull(device); + Assert.assertEquals("Edge Device 2 Updated", device.getName()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java index 83b2370cf0..4be82e7d8b 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RuleChainEdgeTest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.edge; -import com.google.protobuf.AbstractMessage; +import com.datastax.oss.driver.api.core.uuid.Uuids; import org.junit.Assert; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; @@ -29,11 +29,10 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; -import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import java.util.ArrayList; import java.util.Collections; @@ -49,7 +48,7 @@ public class RuleChainEdgeTest extends AbstractEdgeTest { @Test public void testRuleChains() throws Exception { // create rule chain - edgeImitator.expectMessageAmount(2); + edgeImitator.expectMessageAmount(4); RuleChain ruleChain = new RuleChain(); ruleChain.setName("Edge Test Rule Chain"); ruleChain.setType(RuleChainType.EDGE); @@ -67,8 +66,6 @@ public class RuleChainEdgeTest extends AbstractEdgeTest { Assert.assertEquals(ruleChainUpdateMsg.getIdLSB(), savedRuleChain.getUuidId().getLeastSignificantBits()); Assert.assertEquals(ruleChainUpdateMsg.getName(), savedRuleChain.getName()); - testRuleChainMetadataRequestMsg(savedRuleChain.getId()); - // unassign rule chain from edge edgeImitator.expectMessageAmount(1); doDelete("/api/edge/" + edge.getUuidId() @@ -89,55 +86,56 @@ public class RuleChainEdgeTest extends AbstractEdgeTest { } @Test - public void testSendRuleChainMetadataRequestToCloud() throws Exception { - RuleChainId edgeRootRuleChainId = edge.getRootRuleChainId(); + public void testRuleChainToCloud() throws Exception { + UUID uuid = Uuids.timeBased(); + // create rule chain on edge UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); - RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder(); - ruleChainMetadataRequestMsgBuilder.setRuleChainIdMSB(edgeRootRuleChainId.getId().getMostSignificantBits()); - ruleChainMetadataRequestMsgBuilder.setRuleChainIdLSB(edgeRootRuleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); - uplinkMsgBuilder.addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build()); + RuleChainUpdateMsg.Builder ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder(); + ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + ruleChainUpdateMsgBuilder.setName("Rule Chain Edge"); + ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder); + uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build()); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); edgeImitator.expectResponsesAmount(1); - edgeImitator.expectMessageAmount(1); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage; - Assert.assertEquals(ruleChainMetadataUpdateMsg.getRuleChainIdMSB(), edgeRootRuleChainId.getId().getMostSignificantBits()); - Assert.assertEquals(ruleChainMetadataUpdateMsg.getRuleChainIdLSB(), edgeRootRuleChainId.getId().getLeastSignificantBits()); + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataUpdateMsg); - } + RuleChain ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class); + Assert.assertNotNull(ruleChain); + Assert.assertEquals("Rule Chain Edge", ruleChain.getName()); - private void testRuleChainMetadataRequestMsg(RuleChainId ruleChainId) throws Exception { - RuleChainMetadataRequestMsg.Builder ruleChainMetadataRequestMsgBuilder = RuleChainMetadataRequestMsg.newBuilder() - .setRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits()) - .setRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits()); - testAutoGeneratedCodeByProtobuf(ruleChainMetadataRequestMsgBuilder); + // update rule chain on edge + uplinkMsgBuilder = UplinkMsg.newBuilder(); + ruleChainUpdateMsgBuilder = RuleChainUpdateMsg.newBuilder(); + ruleChainUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + ruleChainUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + ruleChainUpdateMsgBuilder.setName("Rule Chain Edge Updated"); + ruleChainUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(ruleChainUpdateMsgBuilder); + uplinkMsgBuilder.addRuleChainUpdateMsg(ruleChainUpdateMsgBuilder.build()); - UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder() - .addRuleChainMetadataRequestMsg(ruleChainMetadataRequestMsgBuilder.build()); testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); edgeImitator.expectResponsesAmount(1); - edgeImitator.expectMessageAmount(1); edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); - Assert.assertTrue(edgeImitator.waitForMessages()); - AbstractMessage latestMessage = edgeImitator.getLatestMessage(); - Assert.assertTrue(latestMessage instanceof RuleChainMetadataUpdateMsg); - RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = (RuleChainMetadataUpdateMsg) latestMessage; - RuleChainId receivedRuleChainId = - new RuleChainId(new UUID(ruleChainMetadataUpdateMsg.getRuleChainIdMSB(), ruleChainMetadataUpdateMsg.getRuleChainIdLSB())); - Assert.assertEquals(ruleChainId, receivedRuleChainId); + latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + ruleChain = doGet("/api/ruleChain/" + uuid, RuleChain.class); + Assert.assertNotNull(ruleChain); + Assert.assertEquals("Rule Chain Edge Updated", ruleChain.getName()); } private void createRuleChainMetadata(RuleChain ruleChain) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 95cdc0e02d..94e0979ef5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -44,6 +44,8 @@ public interface RuleChainService extends EntityDaoService { RuleChain saveRuleChain(RuleChain ruleChain); + RuleChain saveRuleChain(RuleChain ruleChain, boolean doValidate); + boolean setRootRuleChain(TenantId tenantId, RuleChainId ruleChainId); RuleChainUpdateResult saveRuleChainMetaData(TenantId tenantId, RuleChainMetaData ruleChainMetaData, Function ruleNodeUpdater); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index d9da289001..f7d5b5d771 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -420,6 +420,7 @@ message TenantProfileUpdateMsg { bytes profileDataBytes = 8; } +// deprecated message RuleChainMetadataRequestMsg { int64 ruleChainIdMSB = 1; int64 ruleChainIdLSB = 2; @@ -557,7 +558,7 @@ message UplinkMsg { repeated DeviceCredentialsUpdateMsg deviceCredentialsUpdateMsg = 4; repeated AlarmUpdateMsg alarmUpdateMsg = 5; repeated RelationUpdateMsg relationUpdateMsg = 6; - repeated RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg = 7; + repeated RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg = 7; // deprecated repeated AttributesRequestMsg attributesRequestMsg = 8; repeated RelationRequestMsg relationRequestMsg = 9; repeated UserCredentialsRequestMsg userCredentialsRequestMsg = 10; @@ -571,6 +572,8 @@ message UplinkMsg { repeated EntityViewUpdateMsg entityViewUpdateMsg = 18; repeated AssetProfileUpdateMsg assetProfileUpdateMsg = 19; repeated DeviceProfileUpdateMsg deviceProfileUpdateMsg = 20; + repeated RuleChainUpdateMsg ruleChainUpdateMsg = 21; + repeated RuleChainMetadataUpdateMsg ruleChainMetadataUpdateMsg = 22; } message UplinkResponseMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 5e6b1e8009..387fc00c6c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -108,7 +108,20 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC @Override @Transactional public RuleChain saveRuleChain(RuleChain ruleChain) { - ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); + return doSaveRuleChain(ruleChain, true); + } + + @Override + @Transactional + public RuleChain saveRuleChain(RuleChain ruleChain, boolean doValidate) { + return doSaveRuleChain(ruleChain, doValidate); + } + + private RuleChain doSaveRuleChain(RuleChain ruleChain, boolean doValidate) { + log.trace("Executing doSaveRuleChain [{}]", ruleChain); + if (doValidate) { + ruleChainValidator.validate(ruleChain, RuleChain::getTenantId); + } try { RuleChain savedRuleChain = ruleChainDao.save(ruleChain.getTenantId(), ruleChain); if (ruleChain.getId() == null) { From 2ef441d115fe861dc5ba8ea31ccd6c9780811375 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 8 Sep 2023 15:27:49 +0300 Subject: [PATCH 002/438] Tmp check test logic for rule chain --- .../thingsboard/server/dao/rule/BaseRuleChainService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 387fc00c6c..7e9ae12035 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -178,11 +178,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (nodes != null) { for (RuleNode node : nodes) { setSingletonMode(node); + /* TODO: voba - merge comment if (node.getId() != null) { ruleNodeIndexMap.put(node.getId(), nodes.indexOf(node)); } else { toAddOrUpdate.add(node); } + */ + ruleNodeIndexMap.put(node.getId(), nodes.indexOf(node)); + toAddOrUpdate.add(node); } } @@ -193,8 +197,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { - newRuleNode = ruleChainMetaData.getNodes().get(index); - toAddOrUpdate.add(newRuleNode); +// newRuleNode = ruleChainMetaData.getNodes().get(index); +// toAddOrUpdate.add(newRuleNode); } else { updatedRuleNodes.add(new RuleNodeUpdateResult(existingNode, null)); toDelete.add(existingNode); From 055fab9ad54a789ecb6451692e6a57c85b2e02bd Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 8 Sep 2023 17:09:06 +0300 Subject: [PATCH 003/438] Fix test: missing bean and improve safe for rule chain --- .../edge/rpc/processor/rule/BaseRuleChainProcessor.java | 4 ++-- .../edge/rpc/processor/rule/RuleChainEdgeProcessor.java | 3 ++- .../service/edge/rpc/processor/BaseEdgeProcessorTest.java | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java index dc9fa6e14d..66a471e3a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/BaseRuleChainProcessor.java @@ -41,7 +41,7 @@ import java.util.function.Function; @Slf4j public class BaseRuleChainProcessor extends BaseEdgeProcessor { - protected boolean saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg) { + protected boolean saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, RuleChainType ruleChainType) { boolean created = false; RuleChain ruleChain = ruleChainService.findRuleChainById(tenantId, ruleChainId); if (ruleChain == null) { @@ -51,7 +51,7 @@ public class BaseRuleChainProcessor extends BaseEdgeProcessor { ruleChain.setCreatedTime(Uuids.unixTimestamp(ruleChainId.getId())); } ruleChain.setName(ruleChainUpdateMsg.getName()); - ruleChain.setType(RuleChainType.EDGE); + ruleChain.setType(ruleChainType); ruleChain.setDebugMode(ruleChainUpdateMsg.getDebugMode()); ruleChain.setConfiguration(JacksonUtil.toJsonNode(ruleChainUpdateMsg.getConfiguration())); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java index ae5e158ba7..e2ce1fe651 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/rule/RuleChainEdgeProcessor.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -105,7 +106,7 @@ public class RuleChainEdgeProcessor extends BaseRuleChainProcessor { } private void saveOrUpdateRuleChain(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateMsg ruleChainUpdateMsg, Edge edge) { - boolean created = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg); + boolean created = super.saveOrUpdateRuleChain(tenantId, ruleChainId, ruleChainUpdateMsg, RuleChainType.EDGE); if (created) { createRelationFromEdge(tenantId, edge.getId(), ruleChainId); pushRuleChainCreatedEventToRuleEngine(tenantId, edge, ruleChainId); diff --git a/application/src/test/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorTest.java b/application/src/test/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorTest.java index 1c4b481302..9519362305 100644 --- a/application/src/test/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorTest.java +++ b/application/src/test/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessorTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; @@ -196,6 +197,9 @@ public abstract class BaseEdgeProcessorTest { @MockBean protected DataValidator entityViewValidator; + @MockBean + protected DataValidator ruleChainValidator; + @MockBean protected EdgeMsgConstructor edgeMsgConstructor; From 91ff77d20bc1e445eca314077fe77a3065657f8f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:42:48 +0300 Subject: [PATCH 004/438] Git Check --- .../main/resources/sql/schema-entities.sql | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9c95f385f8..c8872cdc15 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,5 +1,5 @@ -- --- Copyright © 2016-2024 The Thingsboard Authors +-- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- -- Licensed under the Apache License, Version 2.0 (the "License"); -- you may not use this file except in compliance with the License. @@ -896,3 +896,31 @@ CREATE TABLE IF NOT EXISTS mobile_app_settings ( qr_code_config VARCHAR(100000), CONSTRAINT mobile_app_settings_tenant_id_unq_key UNIQUE (tenant_id) ); + +CREATE TABLE IF NOT EXISTS calculated_field ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, + type varchar(32) NOT NULL, + name varchar(255) NOT NULL, + configuration_version int DEFAULT 0, + configuration varchar(1000000), + version BIGINT DEFAULT 1, + external_id UUID, + CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(10000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); From fd0af733c1eba5c65fcedf4c4225d87f1983cdc3 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 17 Oct 2024 16:59:08 +0300 Subject: [PATCH 005/438] LICENSE CHECK --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index c8f142f0bf..2717e8bf56 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ +12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From d8fd9234def7fa2dba80d9328e53f887e6aa952d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 30 Oct 2024 17:16:39 +0200 Subject: [PATCH 006/438] Revert "LICENSE CHECK" This reverts commit fd0af733c1eba5c65fcedf4c4225d87f1983cdc3. --- LICENSE | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE b/LICENSE index 2717e8bf56..c8f142f0bf 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,3 @@ -12312321 CONFIDENTIAL Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ From 32bbd835419b00b2e4dd02e973b13d3b00291e53 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 31 Oct 2024 16:42:28 +0200 Subject: [PATCH 007/438] added calculated field entity, dao and controller --- .../server/controller/BaseController.java | 12 ++ .../controller/CalculatedFieldController.java | 95 ++++++++++++++ .../service/security/permission/Resource.java | 4 +- .../permission/TenantAdminPermissions.java | 1 + .../CalculatedFieldService.java | 31 +++++ .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedField.java | 94 ++++++++++++++ .../common/data/id/CalculatedFieldId.java | 44 +++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 75 +++++++++++ .../calculated_field/CalculatedFieldDao.java | 22 ++++ .../server/dao/model/ModelConstants.java | 14 +++ .../dao/model/sql/CalculatedFieldEntity.java | 118 ++++++++++++++++++ .../CalculatedFieldRepository.java | 24 ++++ .../JpaCalculatedFieldDao.java | 47 +++++++ .../main/resources/sql/schema-entities.sql | 47 ++++--- 17 files changed, 617 insertions(+), 17 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index da7d6243fc..10323f4fa0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,6 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -77,6 +78,7 @@ import org.thingsboard.server.common.data.id.AlarmCommentId; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -124,6 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -343,6 +346,9 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; + @Autowired + protected CalculatedFieldService calculatedFieldService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -645,6 +651,8 @@ public abstract class BaseController { case MOBILE_APP: checkMobileAppId(new MobileAppId(entityId.getId()), operation); return; + case CALCULATED_FIELD: + checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } @@ -915,6 +923,10 @@ public abstract class BaseController { } } + protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { + return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); + } + protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java new file mode 100644 index 0000000000..6bb5fc15f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import io.swagger.v3.oas.annotations.Parameter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.config.annotations.ApiOperation; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; + +@RestController +@TbCoreComponent +@RequestMapping("/api") +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldController extends BaseController{ + + private final CalculatedFieldService calculatedFieldService; + + public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", + notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + + "The newly created Calculated Field Id will be present in the response. " + + "Specify existing Calculated Field Id to update the calculated field. " + + "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) + @ResponseBody + public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") + @RequestBody CalculatedField calculatedField) throws Exception { + calculatedField.setTenantId(getTenantId()); + checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + return calculatedFieldService.save(calculatedField); + } + + @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", + notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) + @ResponseBody + public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { + checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); + return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + } + + + @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", + notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { + checkParameter(CALCULATED_FIELD_ID, strCalculatedField); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); + checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 16a3c4be59..7ec040ecd8 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -49,7 +49,9 @@ public enum Resource { VERSION_CONTROL, NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), - MOBILE_APP_SETTINGS; + MOBILE_APP_SETTINGS, + CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + private final Set entityTypes; Resource() { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 10807e4b5a..897581cc91 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -51,6 +51,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java new file mode 100644 index 0000000000..7f79f481b0 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; + +public interface CalculatedFieldService extends EntityDaoService { + + CalculatedField save(CalculatedField calculatedField); + + CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index b5ce79e20f..7b6e99095f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -60,7 +60,8 @@ public enum EntityType { QUEUE_STATS(34), OAUTH2_CLIENT(35), DOMAIN(36), - MOBILE_APP(37); + MOBILE_APP(37), + CALCULATED_FIELD(38); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java new file mode 100644 index 0000000000..d780262726 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { + + private static final long serialVersionUID = 4491966747773381420L; + + private TenantId tenantId; + private EntityId entityId; + + @NoXss + @Length(fieldName = "type") + private String type; + @NoXss + @Length(fieldName = "name") + @Schema(description = "User defined name of the calculated field.") + private String name; + @Schema(description = "Version of calculated field configuration.", example = "0") + private int configurationVersion; + @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + @Getter + @Setter + private Long version; + @Getter + @Setter + private CalculatedFieldId externalId; + + public CalculatedField() { + super(); + } + + public CalculatedField(CalculatedFieldId id) { + super(id); + } + + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { + super(); + this.tenantId = tenantId; + this.entityId = entityId; + this.type = type; + this.name = name; + this.configurationVersion = configurationVersion; + this.configuration = configuration; + this.version = version; + this.externalId = externalId; + } + + @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") + @Override + public CalculatedFieldId getId() { + return super.getId(); + } + + @Schema(description = "Timestamp of the calculated field creation, in milliseconds", example = "1609459200000", accessMode = Schema.AccessMode.READ_ONLY) + @Override + public long getCreatedTime() { + return super.getCreatedTime(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java new file mode 100644 index 0000000000..0de6209b9d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldId.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldId fromString(String calculatedFieldId) { + return new CalculatedFieldId(UUID.fromString(calculatedFieldId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD", allowableValues = "CALCULATED_FIELD") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 5a85e6ce67..2b53011bdd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -111,6 +111,8 @@ public class EntityIdFactory { return new MobileAppId(uuid); case DOMAIN: return new DomainId(uuid); + case CALCULATED_FIELD: + return new CalculatedFieldId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f84c7f7c9b..985e7518f5 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -58,6 +58,7 @@ enum EntityTypeProto { OAUTH2_CLIENT = 35; DOMAIN = 36; MOBILE_APP = 37; + CALCULATED_FIELD = 38; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java new file mode 100644 index 0000000000..f9f7681a3a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Optional; + +import static org.thingsboard.server.dao.service.Validator.validateId; + +@Service("CalculatedFieldDaoService") +@Slf4j +@RequiredArgsConstructor +public class BaseCalculatedFieldService implements CalculatedFieldService { + + public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; + + private final CalculatedFieldDao calculatedFieldDao; + + + @Override + public CalculatedField save(CalculatedField calculatedField) { + log.trace("Executing save, [{}]", calculatedField); + return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + } + + @Override + public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); + } + + @Override + public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); + } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java new file mode 100644 index 0000000000..71decef684 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.Dao; + +public interface CalculatedFieldDao extends Dao { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f85cd4bac4..c37cbfca8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -691,6 +691,20 @@ public class ModelConstants { public static final String MOBILE_APP_SETTINGS_IOS_CONFIG_PROPERTY = "ios_config"; public static final String MOBILE_APP_SETTINGS_QR_CODE_CONFIG_PROPERTY = "qr_code_config"; + /** + * Calculated fields constants. + */ + public static final String CALCULATED_FIELD_TABLE_NAME = "calculated_field"; + public static final String CALCULATED_FIELD_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_TYPE = "type"; + public static final String CALCULATED_FIELD_NAME = "name"; + public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; + public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; + public static final String CALCULATED_FIELD_VERSION = "version"; + public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java new file mode 100644 index 0000000000..35c50c561f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_EXTERNAL_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_TABLE_NAME) +public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_TYPE) + private String type; + + @Column(name = CALCULATED_FIELD_NAME) + private String name; + + @Column(name = CALCULATED_FIELD_CONFIGURATION_VERSION) + private int configurationVersion; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_CONFIGURATION) + private JsonNode configuration; + + @Column(name = CALCULATED_FIELD_VERSION) + private Long version; + + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) + private UUID externalId; + + public CalculatedFieldEntity() { + super(); + } + + public CalculatedFieldEntity(CalculatedField calculatedField) { + this.setUuid(calculatedField.getUuidId()); + this.createdTime = calculatedField.getCreatedTime(); + this.tenantId = calculatedField.getTenantId().getId(); + this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityId = calculatedField.getEntityId().getId(); + this.type = calculatedField.getType(); + this.name = calculatedField.getName(); + this.configurationVersion = calculatedField.getConfigurationVersion(); + this.configuration = calculatedField.getConfiguration(); + this.version = calculatedField.getVersion(); + if (calculatedField.getExternalId() != null) { + this.externalId = calculatedField.getExternalId().getId(); + } + } + + @Override + public CalculatedField toData() { + CalculatedField calculatedField = new CalculatedField(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(configuration); + calculatedField.setVersion(version); + if (externalId != null) { + calculatedField.setExternalId(new CalculatedFieldId(externalId)); + } + return calculatedField; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java new file mode 100644 index 0000000000..79146cf317 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; + +import java.util.UUID; + +public interface CalculatedFieldRepository extends JpaRepository { +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java new file mode 100644 index 0000000000..33cf926eaf --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { + + private final CalculatedFieldRepository calculatedFieldRepository; + + @Override + protected Class getEntityClass() { + return CalculatedFieldEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldRepository; + } +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a26bb03e62..545486180d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -1,3 +1,19 @@ +-- +-- Copyright © 2016-2024 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + -- -- ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL -- @@ -891,6 +907,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, created_time bigint NOT NULL, tenant_id uuid NOT NULL, + entity_type VARCHAR(32), entity_id uuid NOT NULL, type varchar(32) NOT NULL, name varchar(255) NOT NULL, @@ -899,18 +916,18 @@ CREATE TABLE IF NOT EXISTS calculated_field ( version BIGINT DEFAULT 1, external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id) -); - -CREATE TABLE IF NOT EXISTS calculated_field_link ( - id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - entity_id uuid NOT NULL, --- target_id uuid NOT NULL, - calculated_field_id uuid NOT NULL, - configuration varchar(10000), - CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), - CONSTRAINT device_external_id_unq_key UNIQUE (tenant_id, external_id), - CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE -); + CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) +); + +-- CREATE TABLE IF NOT EXISTS calculated_field_link ( +-- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, +-- created_time bigint NOT NULL, +-- tenant_id uuid NOT NULL, +-- entity_id uuid NOT NULL, +-- -- target_id uuid NOT NULL, +-- calculated_field_id uuid NOT NULL, +-- configuration varchar(10000), +-- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), +-- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), +-- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +-- ); From e2d17a822d7519e0d611d06b1ec1abd2160e812c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 1 Nov 2024 13:34:04 +0200 Subject: [PATCH 008/438] added tests --- .../server/controller/BaseController.java | 1 + .../CalculatedFieldControllerTest.java | 143 ++++++++++++++++++ .../calculated_field/CalculatedField.java | 24 ++- .../BaseCalculatedFieldService.java | 1 - .../CalculatedFieldServiceTest.java | 123 +++++++++++++++ 5 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 10323f4fa0..6d52bdb358 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -653,6 +653,7 @@ public abstract class BaseController { return; case CALCULATED_FIELD: checkCalculatedFieldId(new CalculatedFieldId(entityId.getId()), operation); + return; default: checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation); } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java new file mode 100644 index 0000000000..05ffd03970 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class CalculatedFieldControllerTest extends AbstractControllerTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); + + private Tenant savedTenant; + private User tenantAdmin; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + savedTenant = saveTenant(tenant); + assertThat(savedTenant).isNotNull(); + + tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + deleteTenant(savedTenant.getId()); + } + + @Test + public void testSaveCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(savedTenant.getId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testGetCalculatedFieldById() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); + + assertThat(fetchedCalculatedField).isNotNull(); + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testDeleteCalculatedField() throws Exception { + CalculatedField calculatedField = getCalculatedField(); + + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + assertThat(savedCalculatedField).isNotNull(); + + doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) + .andExpect(status().isOk()); + doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); + + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index d780262726..0bd8ded8f4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -21,10 +21,7 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,7 +31,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { private static final long serialVersionUID = 4491966747773381420L; @@ -91,4 +88,21 @@ public class CalculatedField extends BaseData implements HasN return super.getCreatedTime(); } + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedField[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", type='").append(type) + .append(", name='").append(name) + .append(", configurationVersion=").append(configurationVersion) + .append(", configuration=").append(configuration) + .append(", version=").append(version) + .append(", externalId=").append(externalId) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index f9f7681a3a..530a533085 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -39,7 +39,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; - @Override public CalculatedField save(CalculatedField calculatedField) { log.trace("Executing save, [{}]", calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java new file mode 100644 index 0000000000..d2e3d40c5c --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -0,0 +1,123 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.service.AbstractServiceTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldServiceTest extends AbstractServiceTest { + + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); + + @Autowired + private CalculatedFieldService calculatedFieldService; + + private ListeningExecutorService executor; + + @Before + public void before() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); + } + + @After + public void after() { + executor.shutdownNow(); + } + + @Test + public void testSaveCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThat(savedCalculatedField).isNotNull(); + assertThat(savedCalculatedField.getId()).isNotNull(); + assertThat(savedCalculatedField.getCreatedTime()).isGreaterThan(0); + assertThat(savedCalculatedField.getTenantId()).isEqualTo(calculatedField.getTenantId()); + assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); + assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); + assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); + + savedCalculatedField.setName("Test CF"); + + CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); + + assertThat(updatedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testFindCalculatedFieldById() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); + + assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + + @Test + public void testDeleteCalculatedField() { + CalculatedField calculatedField = getCalculatedField(); + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + + assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + + " \"T\": {\n" + + " \"key\": \"temperature\",\n" + + " \"type\": \"TIME_SERIES\"\n" + + " },\n" + + " \"H\": {\n" + + " \"key\": \"humidity\",\n" + + " \"type\": \"TIME_SERIES\",\n" + + " \"defaultValue\": 50\n" + + " }\n" + + " }\n")); + calculatedField.setVersion(1L); + return calculatedField; + } + +} From 03ff7c17ac0b4e7049f9bb7fc060e0a219237a9f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 4 Nov 2024 15:57:37 +0200 Subject: [PATCH 009/438] added calculated field link entity and its dao --- .../controller/CalculatedFieldController.java | 4 +- .../service/security/permission/Resource.java | 3 +- .../CalculatedFieldControllerTest.java | 23 +++-- .../CalculatedFieldService.java | 3 + .../server/common/data/EntityType.java | 3 +- .../calculated_field/CalculatedFieldLink.java | 71 ++++++++++++++ .../common/data/id/CalculatedFieldLinkId.java | 45 +++++++++ .../common/data/id/EntityIdFactory.java | 2 + common/proto/src/main/proto/queue.proto | 1 + .../BaseCalculatedFieldService.java | 66 ++++++++++++- .../calculated_field/CalculatedFieldDao.java | 2 +- .../CalculatedFieldLinkDao.java | 27 ++++++ .../server/dao/model/ModelConstants.java | 10 ++ .../model/sql/CalculatedFieldLinkEntity.java | 92 +++++++++++++++++++ .../CalculatedFieldDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkDataValidator.java | 41 +++++++++ .../CalculatedFieldLinkRepository.java | 27 ++++++ .../JpaCalculatedFieldLinkDao.java | 54 +++++++++++ .../main/resources/sql/schema-entities.sql | 24 ++--- .../CalculatedFieldServiceTest.java | 90 ++++++++++++++++-- .../CalculatedFieldDataValidatorTest.java | 57 ++++++++++++ .../CalculatedFieldLinkDataValidatorTest.java | 57 ++++++++++++ 22 files changed, 702 insertions(+), 41 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 6bb5fc15f9..26a3539a46 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -44,7 +44,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @RequestMapping("/api") @RequiredArgsConstructor @Slf4j -public class CalculatedFieldController extends BaseController{ +public class CalculatedFieldController extends BaseController { private final CalculatedFieldService calculatedFieldService; @@ -61,7 +61,7 @@ public class CalculatedFieldController extends BaseController{ @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") - @RequestBody CalculatedField calculatedField) throws Exception { + @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java index 7ec040ecd8..f57f460fc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java @@ -50,7 +50,8 @@ public enum Resource { NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE, EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE), MOBILE_APP_SETTINGS, - CALCULATED_FIELD(EntityType.CALCULATED_FIELD); + CALCULATED_FIELD(EntityType.CALCULATED_FIELD), + CALCULATED_FIELD_LINK(EntityType.CALCULATED_FIELD_LINK); private final Set entityTypes; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 05ffd03970..685a58d48c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -19,6 +19,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; @@ -26,18 +27,13 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; -import java.util.UUID; - import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class CalculatedFieldControllerTest extends AbstractControllerTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("9e408b94-dc05-47e2-a21c-1a6c0d7bd90a")); - private Tenant savedTenant; - private User tenantAdmin; @Before public void beforeTest() throws Exception { @@ -48,14 +44,14 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { savedTenant = saveTenant(tenant); assertThat(savedTenant).isNotNull(); - tenantAdmin = new User(); + User tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); - tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + createUserAndLogin(tenantAdmin, "testPassword1"); } @After @@ -67,7 +63,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testSaveCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -93,7 +90,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testGetCalculatedFieldById() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); CalculatedField fetchedCalculatedField = doGet("/api/calculatedField/" + savedCalculatedField.getId().getId(), CalculatedField.class); @@ -107,7 +105,8 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { @Test public void testDeleteCalculatedField() throws Exception { - CalculatedField calculatedField = getCalculatedField(); + Device testDevice = createDevice("Test device", "1234567890"); + CalculatedField calculatedField = getCalculatedField(testDevice.getId()); CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); @@ -119,9 +118,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { } - private CalculatedField getCalculatedField() { + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index 7f79f481b0..e1779e1d33 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -28,4 +29,6 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 7b6e99095f..37d77bd2c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -61,7 +61,8 @@ public enum EntityType { OAUTH2_CLIENT(35), DOMAIN(36), MOBILE_APP(37), - CALCULATED_FIELD(38); + CALCULATED_FIELD(38), + CALCULATED_FIELD_LINK(39); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java new file mode 100644 index 0000000000..79f05c9b58 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Schema +@Data +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldLink extends BaseData { + + private static final long serialVersionUID = 6492846246722091530L; + + private TenantId tenantId; + private EntityId entityId; + + @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) + private CalculatedFieldId calculatedFieldId; + @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) + private transient JsonNode configuration; + + public CalculatedFieldLink() { + super(); + } + + public CalculatedFieldLink(CalculatedFieldLinkId id) { + super(id); + } + + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.configuration = configuration; + this.calculatedFieldId = calculatedFieldId; + } + + @Override + public String toString() { + return new StringBuilder() + .append("CalculatedFieldLink[") + .append("tenantId=").append(tenantId) + .append(", entityId=").append(entityId) + .append(", calculatedFieldId=").append(calculatedFieldId) + .append(", configuration=").append(configuration) + .append(", createdTime=").append(createdTime) + .append(", id=").append(id).append(']') + .toString(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java new file mode 100644 index 0000000000..8817aa8a1e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/CalculatedFieldLinkId.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Schema +public class CalculatedFieldLinkId extends UUIDBased implements EntityId { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public CalculatedFieldLinkId(@JsonProperty("id") UUID id) { + super(id); + } + + public static CalculatedFieldLinkId fromString(String calculatedFieldLinkId) { + return new CalculatedFieldLinkId(UUID.fromString(calculatedFieldLinkId)); + } + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "string", example = "CALCULATED_FIELD_LINK", allowableValues = "CALCULATED_FIELD_LINK") + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD_LINK; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 2b53011bdd..973aa5202e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -113,6 +113,8 @@ public class EntityIdFactory { return new DomainId(uuid); case CALCULATED_FIELD: return new CalculatedFieldId(uuid); + case CALCULATED_FIELD_LINK: + return new CalculatedFieldLinkId(uuid); } throw new IllegalArgumentException("EntityType " + type + " is not supported!"); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 985e7518f5..875a531566 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -59,6 +59,7 @@ enum EntityTypeProto { DOMAIN = 36; MOBILE_APP = 37; CALCULATED_FIELD = 38; + CALCULATED_FIELD_LINK = 39; } /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java index 530a533085..e7bc3be314 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java @@ -20,13 +20,21 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.service.DataValidator; +import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; @Service("CalculatedFieldDaoService") @@ -38,11 +46,28 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; private final CalculatedFieldDao calculatedFieldDao; + private final CalculatedFieldLinkDao calculatedFieldLinkDao; + private final DeviceService deviceService; + private final AssetService assetService; + private final DataValidator calculatedFieldDataValidator; + private final DataValidator calculatedFieldLinkDataValidator; @Override public CalculatedField save(CalculatedField calculatedField) { - log.trace("Executing save, [{}]", calculatedField); - return calculatedFieldDao.save(calculatedField.getTenantId(), calculatedField); + calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + try { + TenantId tenantId = calculatedField.getTenantId(); + checkEntityExistence(tenantId, calculatedField.getEntityId()); + log.trace("Executing save calculated field, [{}]", calculatedField); + CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); + createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + return savedCalculatedField; + } catch (Exception e) { + checkConstraintViolation(e, + "calculated_field_unq_key", "Calculated Field with such name is already in exists!", + "calculated_field_external_id_unq_key", "Calculated Field with such external id already exists!"); + throw e; + } } @Override @@ -61,6 +86,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); + try { + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); + } catch (Exception e) { + checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); + throw e; + } + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -71,4 +108,29 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + } + } + + private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); + saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + } + + private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + return calculatedFieldLink; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index 71decef684..e4bd019b18 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -18,5 +18,5 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.dao.Dao; -public interface CalculatedFieldDao extends Dao { +public interface CalculatedFieldDao extends Dao { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java new file mode 100644 index 0000000000..1c422311eb --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.Dao; + +import java.util.UUID; + +public interface CalculatedFieldLinkDao extends Dao { + + CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index c37cbfca8e..368478db6b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -705,6 +705,16 @@ public class ModelConstants { public static final String CALCULATED_FIELD_VERSION = "version"; public static final String CALCULATED_FIELD_EXTERNAL_ID = "external_id"; + /** + * Calculated field links constants. + */ + public static final String CALCULATED_FIELD_LINK_TABLE_NAME = "calculated_field_link"; + public static final String CALCULATED_FIELD_LINK_TENANT_ID_COLUMN = TENANT_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_TYPE = ENTITY_TYPE_COLUMN; + public static final String CALCULATED_FIELD_LINK_ENTITY_ID = ENTITY_ID_COLUMN; + public static final String CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID = "calculated_field_id"; + public static final String CALCULATED_FIELD_LINK_CONFIGURATION = "configuration"; + protected static final String[] NONE_AGGREGATION_COLUMNS = new String[]{LONG_VALUE_COLUMN, DOUBLE_VALUE_COLUMN, BOOLEAN_VALUE_COLUMN, STRING_VALUE_COLUMN, JSON_VALUE_COLUMN, KEY_COLUMN, TS_COLUMN}; protected static final String[] COUNT_AGGREGATION_COLUMNS = new String[]{count(LONG_VALUE_COLUMN), count(DOUBLE_VALUE_COLUMN), count(BOOLEAN_VALUE_COLUMN), count(STRING_VALUE_COLUMN), count(JSON_VALUE_COLUMN), max(TS_COLUMN)}; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java new file mode 100644 index 0000000000..9f2efb230b --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import com.fasterxml.jackson.databind.JsonNode; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.util.mapping.JsonConverter; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_TYPE; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_TENANT_ID_COLUMN; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_LINK_TABLE_NAME) +public class CalculatedFieldLinkEntity extends BaseSqlEntity implements BaseEntity { + + @Column(name = CALCULATED_FIELD_LINK_TENANT_ID_COLUMN) + private UUID tenantId; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_TYPE) + private EntityType entityType; + + @Column(name = CALCULATED_FIELD_LINK_ENTITY_ID) + private UUID entityId; + + @Column(name = CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID) + private UUID calculatedFieldId; + + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_LINK_CONFIGURATION) + private JsonNode configuration; + + + public CalculatedFieldLinkEntity() { + super(); + } + + public CalculatedFieldLinkEntity(CalculatedFieldLink calculatedFieldLink) { + this.setUuid(calculatedFieldLink.getUuidId()); + this.createdTime = calculatedFieldLink.getCreatedTime(); + this.tenantId = calculatedFieldLink.getTenantId().getId(); + this.entityType = calculatedFieldLink.getEntityId().getEntityType(); + this.entityId = calculatedFieldLink.getEntityId().getId(); + this.calculatedFieldId = calculatedFieldLink.getCalculatedFieldId().getId(); + this.configuration = calculatedFieldLink.getConfiguration(); + } + + @Override + public CalculatedFieldLink toData() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(TenantId.fromUUID(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(configuration); + return calculatedFieldLink; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java new file mode 100644 index 0000000000..e50f2c902a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldDao calculatedFieldDao; + + @Override + protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { + CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java new file mode 100644 index 0000000000..83c30d4d03 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.service.DataValidator; + +@Component +public class CalculatedFieldLinkDataValidator extends DataValidator { + + @Autowired + private CalculatedFieldLinkDao calculatedFieldLinkDao; + + @Override + protected CalculatedFieldLink validateUpdate(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { + CalculatedFieldLink old = calculatedFieldLinkDao.findById(calculatedFieldLink.getTenantId(), calculatedFieldLink.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing calculated field link!"); + } + return old; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java new file mode 100644 index 0000000000..1339cf4478 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; + +import java.util.UUID; + +public interface CalculatedFieldLinkRepository extends JpaRepository { + + CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java new file mode 100644 index 0000000000..0721e08d7f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.calculated_field; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.util.SqlDao; + +import java.util.UUID; + +@Slf4j +@Component +@AllArgsConstructor +@SqlDao +public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { + + private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + + @Override + public CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId) { + return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndEntityId(tenantId, entityId)); + } + + @Override + protected Class getEntityClass() { + return CalculatedFieldLinkEntity.class; + } + + @Override + protected JpaRepository getRepository() { + return calculatedFieldLinkRepository; + } + +} diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 545486180d..790317fcb0 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -919,15 +919,15 @@ CREATE TABLE IF NOT EXISTS calculated_field ( CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) ); --- CREATE TABLE IF NOT EXISTS calculated_field_link ( --- id uuid NOT NULL CONSTRAINT calculated_field_pkey PRIMARY KEY, --- created_time bigint NOT NULL, --- tenant_id uuid NOT NULL, --- entity_id uuid NOT NULL, --- -- target_id uuid NOT NULL, --- calculated_field_id uuid NOT NULL, --- configuration varchar(10000), --- CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), --- CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id), --- CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE --- ); +CREATE TABLE IF NOT EXISTS calculated_field_link ( + id uuid NOT NULL CONSTRAINT calculated_field_link_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + entity_type VARCHAR(32), + entity_id uuid NOT NULL, +-- target_id uuid NOT NULL, + calculated_field_id uuid NOT NULL, + configuration varchar(1000000), + CONSTRAINT calculated_field_link_unq_key UNIQUE (entity_id, calculated_field_id), + CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE +); diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java index d2e3d40c5c..fdff4bc2f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java @@ -23,22 +23,28 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.AbstractServiceTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CalculatedFieldServiceTest extends AbstractServiceTest { - private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("71c73816-361e-4e57-82ab-e1deaa8b7d66")); - @Autowired private CalculatedFieldService calculatedFieldService; + @Autowired + private DeviceService deviceService; private ListeningExecutorService executor; @@ -54,7 +60,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -77,10 +84,42 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { } @Test - public void testFindCalculatedFieldById() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + public void testSaveCalculatesFieldWithNonExistingDeviceId() { + CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); + } + + @Test + public void testSaveCalculatedFieldWithExistingName() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such name is already in exists!"); + } + + @Test + public void testSaveCalculatedFieldWithExistingExternalId() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); + + calculatedFieldService.save(calculatedField); + + calculatedField.setName("Test 2"); + assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field with such external id already exists!"); + } + @Test + public void testFindCalculatedFieldById() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); CalculatedField fetchedCalculatedField = calculatedFieldService.findById(tenantId, savedCalculatedField.getId()); assertThat(fetchedCalculatedField).isEqualTo(savedCalculatedField); @@ -90,18 +129,33 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testDeleteCalculatedField() { - CalculatedField calculatedField = getCalculatedField(); - CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + CalculatedField savedCalculatedField = saveValidCalculatedField(); calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); assertThat(calculatedFieldService.findById(tenantId, savedCalculatedField.getId())).isNull(); } - private CalculatedField getCalculatedField() { + @Test + public void testSaveCalculatedFieldLinkIfCalculatedFieldForSuchEntityExists() { + CalculatedField savedCalculatedField = saveValidCalculatedField(); + CalculatedFieldLink calculatedFieldLink = getCalculatedFieldLink(savedCalculatedField); + + assertThatThrownBy(() -> calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Calculated Field for such entity id is already exists!"); + } + + private CalculatedField saveValidCalculatedField() { + Device device = createTestDevice(); + CalculatedField calculatedField = getCalculatedField(device.getId()); + return calculatedFieldService.save(calculatedField); + } + + private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(DEVICE_ID); + calculatedField.setEntityId(deviceId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); @@ -120,4 +174,20 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } + private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + calculatedFieldLink.setEntityId(calculatedField.getEntityId()); + calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); + return calculatedFieldLink; + } + + private Device createTestDevice() { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test"); + return deviceService.saveDevice(device); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java new file mode 100644 index 0000000000..bc7990ca64 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldDataValidator.class) +public class CalculatedFieldDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("7b5229e9-166e-41a9-a257-3b1dafad1b04")); + private final CalculatedFieldId CALCULATED_FIELD_ID = new CalculatedFieldId(UUID.fromString("060fbe45-fbb2-4549-abf3-f72a6be3cb9f")); + + @MockBean + private CalculatedFieldDao calculatedFieldDao; + @SpyBean + private CalculatedFieldDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); + calculatedField.setType("Simple"); + calculatedField.setName("Test"); + + given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedField)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field!"); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java new file mode 100644 index 0000000000..a74c59d7ad --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service.validator; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.BDDMockito.given; + +@SpringBootTest(classes = CalculatedFieldLinkDataValidator.class) +public class CalculatedFieldLinkDataValidatorTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("2ba09d99-6143-43dc-b645-381fc0c43ebe")); + private final CalculatedFieldLinkId CALCULATED_FIELD_LINK_ID = new CalculatedFieldLinkId(UUID.fromString("a5609ef4-cb42-43ce-9b23-e090a4878d1c")); + + @MockBean + private CalculatedFieldLinkDao calculatedFieldLinkDao; + @SpyBean + private CalculatedFieldLinkDataValidator validator; + + @Test + public void testUpdateNonExistingCalculatedField() { + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(CALCULATED_FIELD_LINK_ID); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(UUID.fromString("136477af-fd07-4498-b9c9-54fe50e82992"))); + + given(calculatedFieldLinkDao.findById(TENANT_ID, CALCULATED_FIELD_LINK_ID.getId())).willReturn(null); + + assertThatThrownBy(() -> validator.validateUpdate(TENANT_ID, calculatedFieldLink)) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't update non existing calculated field link!"); + } + +} \ No newline at end of file From ac20551bf41d43a5e094f394a9f1d889c36e0b02 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 5 Nov 2024 17:05:10 +0200 Subject: [PATCH 010/438] added support of housekeeper deletion --- .../controller/CalculatedFieldController.java | 13 +++- ...CalculatedFieldsDeletionTaskProcessor.java | 43 +++++++++++ .../permission/CustomerUserPermissions.java | 31 +++++++- .../security/permission/Operation.java | 2 +- .../permission/TenantAdminPermissions.java | 21 +++-- .../CalculatedFieldControllerTest.java | 1 - .../CalculatedFieldService.java | 5 ++ .../data/housekeeper/HousekeeperTask.java | 4 + .../data/housekeeper/HousekeeperTaskType.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 4 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 31 +++++++- .../calculated_field/CalculatedFieldDao.java | 9 +++ .../dao/device/DeviceProfileServiceImpl.java | 4 +- .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/entity/AbstractEntityService.java | 5 ++ .../dao/housekeeper/CleanUpService.java | 1 + .../CalculatedFieldRepository.java | 6 ++ .../JpaCalculatedFieldDao.java | 16 ++++ .../dao/service/AssetProfileServiceTest.java | 21 +++++ .../server/dao/service/AssetServiceTest.java | 77 ++++++++++++------- .../CalculatedFieldServiceTest.java | 5 +- .../dao/service/DeviceProfileServiceTest.java | 25 +++++- .../server/dao/service/DeviceServiceTest.java | 20 +++++ .../CalculatedFieldLinkDataValidatorTest.java | 2 +- 25 files changed, 300 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java rename dao/src/test/java/org/thingsboard/server/dao/{calculated_field => service}/CalculatedFieldServiceTest.java (97%) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 26a3539a46..b3f2c018cb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -30,11 +30,11 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -63,7 +63,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); - checkEntity(calculatedField.getId(), calculatedField, Resource.CALCULATED_FIELD); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); return calculatedFieldService.save(calculatedField); } @@ -76,7 +76,10 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - return checkCalculatedFieldId(calculatedFieldId, Operation.READ); + CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + checkNotNull(calculatedField); + checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); + return calculatedField; } @@ -88,7 +91,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - checkCalculatedFieldId(calculatedFieldId, Operation.DELETE); + TenantId tenantId = getTenantId(); + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java new file mode 100644 index 0000000000..18d84b02ce --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CalculatedFieldsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final CalculatedFieldService calculatedFieldService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = calculatedFieldService.deleteAllCalculatedFieldsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} calculated fields", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_CALCULATED_FIELDS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 46cd1c2979..0c6dbc1599 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityPermissionChecker); - put(Resource.DEVICE, customerEntityPermissionChecker); + put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,6 +85,29 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; + private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = + new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, + Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, + Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, + Operation.WRITE_CALCULATED_FIELD) { + + @Override + @SuppressWarnings("unchecked") + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + + if (!super.hasPermission(user, operation, entityId, entity)) { + return false; + } + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + if (!(entity instanceof HasCustomerId)) { + return false; + } + return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); + } + }; + private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { @@ -188,7 +211,8 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( + Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { @Override @SuppressWarnings("unchecked") @@ -202,4 +226,5 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; + } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 9465a91cc8..d029d78cff 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -19,6 +19,6 @@ public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, - ASSIGN_TO_TENANT + ASSIGN_TO_TENANT, READ_CALCULATED_FIELD, WRITE_CALCULATED_FIELD } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 897581cc91..c730637148 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -23,15 +23,15 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.model.SecurityUser; -@Component(value="tenantAdminPermissions") +@Component(value = "tenantAdminPermissions") public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityPermissionChecker); - put(Resource.DEVICE, tenantEntityPermissionChecker); + put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -51,14 +51,23 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker); put(Resource.NOTIFICATION, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_SETTINGS, new PermissionChecker.GenericPermissionChecker(Operation.READ)); - put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { + if (!user.getTenantId().equals(entity.getTenantId())) { + return false; + } + return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); + } + }; + + public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { + @Override + public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { return false; } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 685a58d48c..d9f4ac0377 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -115,7 +115,6 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) .andExpect(status().isOk()); doGet("/api/calculatedField/" + savedCalculatedField.getId().getId()).andExpect(status().isNotFound()); - } private CalculatedField getCalculatedField(DeviceId deviceId) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index e1779e1d33..f271f4ed24 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -29,6 +30,10 @@ public interface CalculatedFieldService extends EntityDaoService { void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 8757b1121e..7a87528553 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -81,6 +81,10 @@ public class HousekeeperTask implements Serializable { return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } + public static HousekeeperTask deleteCalculatedFields(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_CALCULATED_FIELDS); + } + @JsonIgnore public String getDescription() { return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index edd13b3d5d..236844c17d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -30,7 +30,8 @@ public enum HousekeeperTaskType { DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"), DELETE_TENANT_ENTITIES("tenant entities deletion"), - DELETE_ENTITIES("entities deletion"); + DELETE_ENTITIES("entities deletion"), + DELETE_CALCULATED_FIELDS("calculated fields deletion"); private final String description; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7d4d5300f9..baf20219ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -72,7 +79,7 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing findById, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); @@ -80,12 +87,21 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteRpc, tenantId [{}], rpcId [{}]", tenantId, calculatedFieldId); + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); } + @Override + public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(entityId.getId(), id -> "Incorrect entityId " + id); + List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); + return calculatedFields.size(); + } + @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); @@ -98,6 +114,11 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -114,6 +135,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index e4bd019b18..d0d0a3c386 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -16,7 +16,16 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldDao extends Dao { + + boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + + List removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 09e2be01e0..84dc1a7d14 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService { + + boolean existsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 33cf926eaf..9a0eef26c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -15,16 +15,21 @@ */ package org.thingsboard.server.dao.sql.calculated_field; +import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -35,6 +40,17 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldRepository.removeAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override protected Class getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java index 7271a0fc8a..c11d8c6e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetProfileServiceTest.java @@ -28,10 +28,12 @@ import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; @@ -43,6 +45,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class AssetProfileServiceTest extends AbstractServiceTest { @@ -54,6 +57,8 @@ public class AssetProfileServiceTest extends AbstractServiceTest { AssetProfileService assetProfileService; @Autowired AssetService assetService; + @Autowired + private CalculatedFieldService calculatedFieldService; @Test public void testSaveAssetProfile() { @@ -380,5 +385,21 @@ public class AssetProfileServiceTest extends AbstractServiceTest { assertThat(assetProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteAssetProfileIfCalculatedFieldExists() { + AssetProfile assetProfile = this.createAssetProfile(tenantId, "Asset Profile"); + AssetProfile savedAssetProfile = assetProfileService.saveAssetProfile(assetProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAssetProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Asset Profile is prohibited!"); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 1976f173a5..e1e1bf70ed 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -39,6 +40,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -47,6 +49,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @DaoSqlTest @@ -63,6 +66,8 @@ public class AssetServiceTest extends AbstractServiceTest { @Autowired private AssetProfileService assetProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; private IdComparator idComparator = new IdComparator<>(); @@ -214,24 +219,24 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); try { - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset B"+i); + asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset C"+i); + asset.setName("My asset C" + i); asset.setType("typeC"); assets.add(assetService.saveAsset(asset)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("My asset A"+i); + asset.setName("My asset A" + i); asset.setType("typeA"); assets.add(assetService.saveAsset(asset)); } @@ -267,10 +272,10 @@ public class AssetServiceTest extends AbstractServiceTest { @Test public void testFindAssetsByTenantId() { List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); assets.add(assetService.saveAsset(asset)); } @@ -303,11 +308,11 @@ public class AssetServiceTest extends AbstractServiceTest { public void testFindAssetsByTenantIdAndName() { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -315,11 +320,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -381,11 +386,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeA"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -394,11 +399,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeB"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -464,10 +469,10 @@ public class AssetServiceTest extends AbstractServiceTest { CustomerId customerId = customer.getId(); List assets = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); - asset.setName("Asset"+i); + asset.setName("Asset" + i); asset.setType("default"); asset = assetService.saveAsset(asset); assets.add(new AssetInfo(assetService.assignAssetToCustomer(tenantId, asset.getId(), customerId), customer.getTitle(), customer.isPublic(), "default")); @@ -508,11 +513,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; List assetsTitle1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -521,11 +526,11 @@ public class AssetServiceTest extends AbstractServiceTest { } String title2 = "Asset title 2"; List assetsTitle2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType("default"); @@ -596,11 +601,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title1 = "Asset title 1"; String type1 = "typeC"; List assetsType1 = new ArrayList<>(); - for (int i=0;i<17;i++) { + for (int i = 0; i < 17; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type1); @@ -610,11 +615,11 @@ public class AssetServiceTest extends AbstractServiceTest { String title2 = "Asset title 2"; String type2 = "typeD"; List assetsType2 = new ArrayList<>(); - for (int i=0;i<13;i++) { + for (int i = 0; i < 13; i++) { Asset asset = new Asset(); asset.setTenantId(tenantId); String suffix = StringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); asset.setName(name); asset.setType(type2); @@ -848,4 +853,24 @@ public class AssetServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteAssetIfCalculatedFieldExists() { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete asset that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java similarity index 97% rename from dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java rename to dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index fdff4bc2f6..46cc478904 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.service; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -28,10 +28,9 @@ import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.service.AbstractServiceTest; -import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index aa14e20151..9bb1abc3c8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -31,9 +31,11 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; @@ -49,6 +51,7 @@ import java.util.concurrent.Executors; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; @DaoSqlTest @@ -60,6 +63,9 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; + @Autowired + private CalculatedFieldService calculatedFieldService; + private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -397,7 +403,7 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { var profileA = deviceProfileService.saveDeviceProfile( - createDeviceProfile(tenantId, "profile A")); + createDeviceProfile(tenantId, "profile A")); deviceProfiles.add(deviceProfileService.saveDeviceProfile(profileA)); @@ -478,4 +484,21 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } + @Test + public void testDeleteDeviceProfileIfCalculatedFieldExists() { + DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedDeviceProfile.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Deletion of Device Profile is prohibited!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 9fe3f85463..f2508090b2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; @@ -87,6 +89,8 @@ public class DeviceServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + private CalculatedFieldService calculatedFieldService; + @Autowired private PlatformTransactionManager platformTransactionManager; @SpyBean private DeviceCredentialsDataValidator validator; @@ -1198,4 +1202,20 @@ public class DeviceServiceTest extends AbstractServiceTest { ); } + @Test + public void testDeleteDeviceIfCalculatedFieldExists() { + Device device = saveDevice(tenantId, "Test"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(device.getId()); + calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete device that has entity views or calculated fields!"); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index a74c59d7ad..c9fa57a751 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -54,4 +54,4 @@ public class CalculatedFieldLinkDataValidatorTest { .hasMessage("Can't update non existing calculated field link!"); } -} \ No newline at end of file +} From c223f1d2ba6754b739ba36f29f1468c4bea2d684 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:06:05 +0200 Subject: [PATCH 011/438] declined permissions for customers --- .../permission/CustomerUserPermissions.java | 27 ++----------------- .../permission/TenantAdminPermissions.java | 19 +++---------- .../dao/asset/AssetProfileServiceImpl.java | 4 +-- .../dao/device/DeviceProfileServiceImpl.java | 4 +-- .../dao/service/AssetProfileServiceTest.java | 22 --------------- .../dao/service/DeviceProfileServiceTest.java | 20 -------------- 6 files changed, 10 insertions(+), 86 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 0c6dbc1599..90019968df 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -34,8 +34,8 @@ public class CustomerUserPermissions extends AbstractPermissions { public CustomerUserPermissions() { super(); put(Resource.ALARM, customerAlarmPermissionChecker); - put(Resource.ASSET, customerEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, customerEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, customerEntityPermissionChecker); + put(Resource.DEVICE, customerEntityPermissionChecker); put(Resource.CUSTOMER, customerPermissionChecker); put(Resource.DASHBOARD, customerDashboardPermissionChecker); put(Resource.ENTITY_VIEW, customerEntityPermissionChecker); @@ -85,29 +85,6 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker customerEntityWithCalculatedFieldPermissionChecker = - new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_CREDENTIALS, - Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY, Operation.RPC_CALL, Operation.CLAIM_DEVICES, - Operation.WRITE, Operation.WRITE_ATTRIBUTES, Operation.WRITE_TELEMETRY, Operation.READ_CALCULATED_FIELD, - Operation.WRITE_CALCULATED_FIELD) { - - @Override - @SuppressWarnings("unchecked") - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - - if (!super.hasPermission(user, operation, entityId, entity)) { - return false; - } - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - if (!(entity instanceof HasCustomerId)) { - return false; - } - return operation.equals(Operation.CLAIM_DEVICES) || user.getCustomerId().equals(((HasCustomerId) entity).getCustomerId()); - } - }; - private static final PermissionChecker customerPermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index c730637148..de11521e85 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -30,8 +30,8 @@ public class TenantAdminPermissions extends AbstractPermissions { super(); put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); - put(Resource.ASSET, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.DEVICE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.ASSET, tenantEntityPermissionChecker); + put(Resource.DEVICE, tenantEntityPermissionChecker); put(Resource.CUSTOMER, tenantEntityPermissionChecker); put(Resource.DASHBOARD, tenantEntityPermissionChecker); put(Resource.ENTITY_VIEW, tenantEntityPermissionChecker); @@ -40,8 +40,8 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.USER, userPermissionChecker); put(Resource.WIDGETS_BUNDLE, widgetsPermissionChecker); put(Resource.WIDGET_TYPE, widgetsPermissionChecker); - put(Resource.DEVICE_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); - put(Resource.ASSET_PROFILE, tenantEntityWithCalculatedFieldPermissionChecker); + put(Resource.DEVICE_PROFILE, tenantEntityPermissionChecker); + put(Resource.ASSET_PROFILE, tenantEntityPermissionChecker); put(Resource.API_USAGE_STATE, tenantEntityPermissionChecker); put(Resource.TB_RESOURCE, tbResourcePermissionChecker); put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker); @@ -55,17 +55,6 @@ public class TenantAdminPermissions extends AbstractPermissions { public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { - @Override - public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { - if (!user.getTenantId().equals(entity.getTenantId())) { - return false; - } - return !Operation.READ_CALCULATED_FIELD.equals(operation) && !Operation.WRITE_CALCULATED_FIELD.equals(operation); - } - }; - - public static final PermissionChecker tenantEntityWithCalculatedFieldPermissionChecker = new PermissionChecker() { - @Override public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) { if (!user.getTenantId().equals(entity.getTenantId())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index baf20219ac..7d4d5300f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -192,8 +192,8 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService assetProfileService.deleteAssetProfile(tenantId, savedAssetProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Asset Profile is prohibited!"); - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java index 9bb1abc3c8..d99eaa2cea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceProfileServiceTest.java @@ -63,9 +63,6 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { DeviceService deviceService; @Autowired OtaPackageService otaPackageService; - @Autowired - private CalculatedFieldService calculatedFieldService; - private IdComparator idComparator = new IdComparator<>(); private IdComparator deviceProfileInfoIdComparator = new IdComparator<>(); @@ -484,21 +481,4 @@ public class DeviceProfileServiceTest extends AbstractServiceTest { assertThat(deviceProfileInfos).isEqualTo(expected); } - @Test - public void testDeleteDeviceProfileIfCalculatedFieldExists() { - DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - CalculatedField calculatedField = new CalculatedField(); - calculatedField.setTenantId(tenantId); - calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); - calculatedField.setEntityId(savedDeviceProfile.getId()); - calculatedFieldService.save(calculatedField); - - assertThatThrownBy(() -> deviceProfileService.deleteDeviceProfile(tenantId, savedDeviceProfile.getId())) - .isInstanceOf(DataValidationException.class) - .hasMessage("Deletion of Device Profile is prohibited!"); - } - } From 3c073bad6e65d7f07e7fee4b1cbc9ecbacf3b9ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 11:11:34 +0200 Subject: [PATCH 012/438] removed authority customer user for endpoints --- .../server/controller/CalculatedFieldController.java | 4 ++-- .../server/dao/device/DeviceProfileServiceImpl.java | 4 ++-- .../server/dao/service/AssetProfileServiceTest.java | 1 + .../server/dao/service/DeviceProfileServiceTest.java | 5 +---- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index b3f2c018cb..78331e61b4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -57,7 +57,7 @@ public class CalculatedFieldController extends BaseController { "Referencing non-existing Calculated Field Id will cause 'Not Found' error. " + "Remove 'id', 'tenantId' from the request body example (below) to create new Calculated Field entity. " + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField", method = RequestMethod.POST) @ResponseBody public CalculatedField saveCalculatedField(@io.swagger.v3.oas.annotations.parameters.RequestBody(description = "A JSON value representing the calculated field.") @@ -70,7 +70,7 @@ public class CalculatedFieldController extends BaseController { @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", notes = "Fetch the Calculated Field object based on the provided Calculated Field Id." ) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") @RequestMapping(value = "/calculatedField/{calculatedFieldId}", method = RequestMethod.GET) @ResponseBody public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index ec68562f5d..09e2be01e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -230,8 +230,8 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService Date: Wed, 6 Nov 2024 15:21:28 +0200 Subject: [PATCH 013/438] added CF config and converter for it --- .../controller/CalculatedFieldController.java | 27 ++++ .../CalculatedFieldControllerTest.java | 38 ++++-- .../CalculatedFieldService.java | 2 + .../calculated_field/CalculatedField.java | 7 +- .../CalculatedFieldConfig.java | 43 +++++++ .../calculated_field/CalculatedFieldLink.java | 8 +- .../server/dao/asset/BaseAssetService.java | 4 +- .../BaseCalculatedFieldService.java | 35 ++++-- .../CalculatedFieldConfigUtil.java | 117 ++++++++++++++++++ .../calculated_field/CalculatedFieldDao.java | 2 + .../CalculatedFieldLinkDao.java | 4 +- .../dao/customer/CustomerServiceImpl.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 6 +- .../model/sql/CalculatedFieldLinkEntity.java | 6 +- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 + .../JpaCalculatedFieldDao.java | 5 + .../JpaCalculatedFieldLinkDao.java | 6 +- .../server/dao/service/AssetServiceTest.java | 40 +++++- .../service/CalculatedFieldServiceTest.java | 52 +++++--- .../dao/service/CustomerServiceTest.java | 57 ++++++++- .../server/dao/service/DeviceServiceTest.java | 34 ++++- 23 files changed, 431 insertions(+), 74 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 78331e61b4..c33fb86ef8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,14 +27,24 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; + +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -46,6 +56,9 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { + private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( + EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); + private final CalculatedFieldService calculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -64,6 +77,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); + checkReferencedEntities(calculatedField.getConfiguration()); return calculatedFieldService.save(calculatedField); } @@ -97,4 +111,17 @@ public class CalculatedFieldController extends BaseController { calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); } + private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { + throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); + } + checkEntityId(referencedEntityId, Operation.READ); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index d9f4ac0377..52271bef83 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -18,15 +18,18 @@ package org.thingsboard.server.controller; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.util.Map; + import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -75,7 +78,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { assertThat(savedCalculatedField.getEntityId()).isEqualTo(calculatedField.getEntityId()); assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); - assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getCalculatedFieldConfig(testDevice.getId())); assertThat(savedCalculatedField.getVersion()).isEqualTo(calculatedField.getVersion()); savedCalculatedField.setName("Test CF"); @@ -123,19 +126,28 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java index f271f4ed24..b60c25e749 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java @@ -36,4 +36,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java index 0bd8ded8f4..2482a7a19a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java @@ -47,8 +47,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema(description = "JSON with the calculated field configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; @Getter @Setter private Long version; @@ -64,8 +64,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, JsonNode configuration, Long version, CalculatedFieldId externalId) { - super(); + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java new file mode 100644 index 0000000000..b1d63a9396 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.calculated_field; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Map; + +@Data +public class CalculatedFieldConfig { + + private Map arguments; + private Output output; + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private int defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java index 79f05c9b58..f24746abc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java @@ -37,8 +37,8 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; - @Schema(description = "JSON with the calculated field link configuration.", implementation = com.fasterxml.jackson.databind.JsonNode.class) - private transient JsonNode configuration; + @Schema + private transient CalculatedFieldConfig configuration; public CalculatedFieldLink() { super(); @@ -48,11 +48,11 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, JsonNode configuration, CalculatedFieldId calculatedFieldId) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { this.tenantId = tenantId; this.entityId = entityId; - this.configuration = configuration; this.calculatedFieldId = calculatedFieldId; + this.configuration = configuration; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index da86d82cf0..f4c68aa15b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -222,8 +222,8 @@ public class BaseAssetService extends AbstractCachedEntityService !referencedEntityId.equals(calculatedField.getEntityId())) + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfig::getArguments) + .flatMap(arguments -> arguments.values().stream()) + .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); @@ -142,22 +153,26 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' is not supported."); + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = calculatedFieldLinkDao.findCalculatedFieldLinkByEntityId(tenantId.getId(), calculatedField.getEntityId().getId()); - saveCalculatedFieldLink(tenantId, Objects.requireNonNullElseGet(calculatedFieldLink, () -> createCalculatedFieldLink(tenantId, calculatedField))); + CalculatedFieldLink existingLink = (calculatedField.getId() != null) + ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) + : null; + + CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); + saveCalculatedFieldLink(tenantId, updatedLink); } - private CalculatedFieldLink createCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); - return calculatedFieldLink; + private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { + CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(calculatedField.getEntityId()); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(calculatedField.getConfiguration()); + return link; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java new file mode 100644 index 0000000000..8f9422b354 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.calculated_field; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +public class CalculatedFieldConfigUtil { + + public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null) { + return null; + } + try { + CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); + Map arguments = new HashMap<>(); + + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + if (argumentNode.has("entityType") && argumentNode.has("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + + if (argumentNode.has("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); + } + + arguments.put(key, argument); + }); + } + calculatedFieldConfig.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + calculatedFieldConfig.setOutput(output); + } + + return calculatedFieldConfig; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); + } + } + + public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { + if (calculatedFieldConfig == null) { + return null; + } + try { + ObjectNode configNode = JacksonUtil.newObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + calculatedFieldConfig.getArguments().forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (calculatedFieldConfig.getOutput() != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", calculatedFieldConfig.getOutput().getType()); + outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); + } + + return configNode; + + } catch (Exception e) { + throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java index d0d0a3c386..e4ec365f46 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java @@ -26,6 +26,8 @@ public interface CalculatedFieldDao extends Dao { boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + List findAllByTenantId(TenantId tenantId); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 1c422311eb..380a3b4d96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -16,12 +16,14 @@ package org.thingsboard.server.dao.calculated_field; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import java.util.UUID; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 2067ef4a5d..955d16c8e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -184,6 +184,10 @@ public class CustomerServiceImpl extends AbstractCachedEntityService implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration(); + this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -107,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(configuration); + calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 9f2efb230b..46c81e4449 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -34,6 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -75,7 +77,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity { - CalculatedFieldLinkEntity findByTenantIdAndEntityId(UUID tenantId, UUID entityId); + CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java index 17d7c0a9c6..d446d59859 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java @@ -25,6 +25,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantId(UUID tenantId); + List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 9a0eef26c3..8a873e1580 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -45,6 +45,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 0721e08d7f..4424f5bf42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -20,6 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -37,8 +39,8 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao { assetService.deleteAsset(tenantId, asset.getId()); }); + assets.forEach((asset) -> { + assetService.deleteAsset(tenantId, asset.getId()); + }); } } @@ -854,23 +858,49 @@ public class AssetServiceTest extends AbstractServiceTest { } @Test - public void testDeleteAssetIfCalculatedFieldExists() { + public void testDeleteAssetIfReferencedInCalculatedField() { Asset asset = new Asset(); asset.setTenantId(tenantId); asset.setName("My asset"); asset.setType("default"); Asset savedAsset = assetService.saveAsset(asset); + Asset assetWithCf = new Asset(); + assetWithCf.setTenantId(tenantId); + assetWithCf.setName("Asset with CF"); + assetWithCf.setType("default"); + Asset savedAssetWithCf = assetService.saveAsset(assetWithCf); + CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(savedAsset.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(savedAssetWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedAsset.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> assetService.deleteAsset(tenantId, savedAsset.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete asset that has entity views or calculated fields!"); + .hasMessage("Can't delete asset that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 46cc478904..32b7cbc00a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -21,17 +21,19 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import java.util.Map; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -60,7 +62,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThat(savedCalculatedField).isNotNull(); @@ -84,7 +86,8 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatesFieldWithNonExistingDeviceId() { - CalculatedField calculatedField = getCalculatedField(new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22"))); + DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); + CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) .isInstanceOf(IllegalArgumentException.class) @@ -94,7 +97,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) @@ -105,7 +108,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { @Test public void testSaveCalculatedFieldWithExistingExternalId() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); calculatedField.setExternalId(new CalculatedFieldId(UUID.fromString("2ef69d0a-89cf-4868-86f8-c50551d87ebe"))); calculatedFieldService.save(calculatedField); @@ -147,28 +150,18 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); - CalculatedField calculatedField = getCalculatedField(device.getId()); + CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); return calculatedFieldService.save(calculatedField); } - private CalculatedField getCalculatedField(DeviceId deviceId) { + private CalculatedField getCalculatedField(EntityId entityId, EntityId referencedEntityId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); - calculatedField.setEntityId(deviceId); + calculatedField.setEntityId(entityId); calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(JacksonUtil.toJsonNode("{\n" + - " \"T\": {\n" + - " \"key\": \"temperature\",\n" + - " \"type\": \"TIME_SERIES\"\n" + - " },\n" + - " \"H\": {\n" + - " \"key\": \"humidity\",\n" + - " \"type\": \"TIME_SERIES\",\n" + - " \"defaultValue\": 50\n" + - " }\n" + - " }\n")); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -177,11 +170,30 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); +// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } + private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(referencedEntityId); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + return config; + } + private Device createTestDevice() { Device device = new Device(); device.setTenantId(tenantId); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0f4c139ce8..c021512a44 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -30,14 +30,20 @@ import org.testcontainers.shaded.org.awaitility.Awaitility; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -45,13 +51,17 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest public class CustomerServiceTest extends AbstractServiceTest { @Autowired CustomerService customerService; + @Autowired + CalculatedFieldService calculatedFieldService; + @Autowired + AssetService assetService; static final int TIMEOUT = 30; @@ -343,4 +353,49 @@ public class CustomerServiceTest extends AbstractServiceTest { } } + @Test + public void testDeleteCustomerIfReferencedInCalculatedField() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("My customer"); + Customer savedCustomer = customerService.saveCustomer(customer); + + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("My asset"); + asset.setType("default"); + Asset savedAsset = assetService.saveAsset(asset); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + calculatedField.setName("Test CF"); + calculatedField.setType("Simple"); + calculatedField.setEntityId(savedAsset.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(savedCustomer.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); + + assertThatThrownBy(() -> customerService.deleteCustomer(tenantId, savedCustomer.getId())) + .isInstanceOf(DataValidationException.class) + .hasMessage("Can't delete customer that is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2508090b2..a9ed4d201f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -66,6 +67,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -1203,19 +1205,41 @@ public class DeviceServiceTest extends AbstractServiceTest { } @Test - public void testDeleteDeviceIfCalculatedFieldExists() { - Device device = saveDevice(tenantId, "Test"); + public void testDeleteAssetIfReferencedInCalculatedField() { + Device device = saveDevice(tenantId, "Test Device"); + Device deviceWithCf = saveDevice(tenantId, "Device with CF"); CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); calculatedField.setType("Simple"); - calculatedField.setEntityId(device.getId()); - calculatedFieldService.save(calculatedField); + calculatedField.setEntityId(deviceWithCf.getId()); + + CalculatedFieldConfig config = new CalculatedFieldConfig(); + + CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + argument.setEntityId(device.getId()); + argument.setType("TIME_SERIES"); + argument.setKey("temperature"); + + config.setArguments(Map.of("T", argument)); + + CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + output.setType("TIME_SERIES"); + output.setExpression("T - (100 - H) / 5"); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + + CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); assertThatThrownBy(() -> deviceService.deleteDevice(tenantId, device.getId())) .isInstanceOf(DataValidationException.class) - .hasMessage("Can't delete device that has entity views or calculated fields!"); + .hasMessage("Can't delete device that has entity views or is referenced in calculated fields!"); + + calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } + } From 47e7c0f3a535aaa16d2d4b9bd74fe9f1f1f556f4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 15:25:07 +0200 Subject: [PATCH 014/438] removed permission for customer --- .../service/security/permission/CustomerUserPermissions.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java index 90019968df..46cd1c2979 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/CustomerUserPermissions.java @@ -188,8 +188,7 @@ public class CustomerUserPermissions extends AbstractPermissions { } }; - private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker( - Operation.READ, Operation.READ_CALCULATED_FIELD, Operation.WRITE_CALCULATED_FIELD) { + private static final PermissionChecker profilePermissionChecker = new PermissionChecker.GenericPermissionChecker(Operation.READ) { @Override @SuppressWarnings("unchecked") @@ -203,5 +202,4 @@ public class CustomerUserPermissions extends AbstractPermissions { return user.getTenantId().equals(entity.getTenantId()); } }; - } From 258ba4807ba0c514091a8bc2f8a475cf6699b701 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 6 Nov 2024 16:29:20 +0200 Subject: [PATCH 015/438] fixed entity service registry test --- .../server/dao/entity/DefaultEntityServiceRegistry.java | 3 +++ .../server/dao/service/EntityServiceRegistryTest.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index 364d3a12f3..56e8b95473 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -43,6 +43,9 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { if (EntityType.RULE_CHAIN.equals(entityType)) { entityDaoServicesMap.put(EntityType.RULE_NODE, entityDaoService); } + if (EntityType.CALCULATED_FIELD.equals(entityType)) { + entityDaoServicesMap.put(EntityType.CALCULATED_FIELD_LINK, entityDaoService); + } }); log.debug("Initialized EntityServiceRegistry total [{}] entries", entityDaoServicesMap.size()); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index c3f6213398..0769479f48 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,6 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; @@ -44,4 +45,8 @@ public class EntityServiceRegistryTest extends AbstractServiceTest { Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.RULE_NODE) instanceof RuleChainService); } + @Test + public void givenCalculatedFieldLinkEntityType_whenGetServiceByEntityTypeCalled_thenReturnedCalculatedFieldService() { + Assert.assertTrue(entityServiceRegistry.getServiceByEntityType(EntityType.CALCULATED_FIELD_LINK) instanceof CalculatedFieldService); + } } From 1d702365381db46e7a925043973ba2d5fe021134 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 09:14:41 +0200 Subject: [PATCH 016/438] fixed entity dao registry test --- .../dao/calculated_field/CalculatedFieldLinkDao.java | 2 -- .../dao/sql/calculated_field/JpaCalculatedFieldDao.java | 7 +++++++ .../sql/calculated_field/JpaCalculatedFieldLinkDao.java | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java index 380a3b4d96..7fa0285b9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java @@ -20,8 +20,6 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; -import java.util.UUID; - public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java index 8a873e1580..affd652514 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java @@ -20,6 +20,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -65,4 +66,10 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getRepository() { return calculatedFieldRepository; } + + @Override + public EntityType getEntityType() { + return EntityType.CALCULATED_FIELD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java index 4424f5bf42..f77baa4e42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -53,4 +54,9 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao Date: Thu, 7 Nov 2024 12:02:39 +0200 Subject: [PATCH 017/438] renamed directories --- .../thingsboard/server/controller/BaseController.java | 4 ++-- .../server/controller/CalculatedFieldController.java | 8 +++----- .../processor/CalculatedFieldsDeletionTaskProcessor.java | 2 +- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldService.java | 6 +++--- .../data/{calculated_field => cf}/CalculatedField.java | 3 +-- .../{calculated_field => cf}/CalculatedFieldConfig.java | 2 +- .../{calculated_field => cf}/CalculatedFieldLink.java | 3 +-- .../BaseCalculatedFieldService.java | 9 ++++----- .../CalculatedFieldConfigUtil.java | 4 ++-- .../dao/{calculated_field => cf}/CalculatedFieldDao.java | 4 ++-- .../{calculated_field => cf}/CalculatedFieldLinkDao.java | 4 ++-- .../server/dao/entity/AbstractEntityService.java | 2 +- .../server/dao/model/sql/CalculatedFieldEntity.java | 6 +++--- .../server/dao/model/sql/CalculatedFieldLinkEntity.java | 6 +++--- .../service/validator/CalculatedFieldDataValidator.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidator.java | 4 ++-- .../CalculatedFieldLinkRepository.java | 2 +- .../CalculatedFieldRepository.java | 2 +- .../{calculated_field => cf}/JpaCalculatedFieldDao.java | 6 +++--- .../JpaCalculatedFieldLinkDao.java | 6 +++--- .../thingsboard/server/dao/service/AssetServiceTest.java | 6 +++--- .../server/dao/service/CalculatedFieldServiceTest.java | 8 ++++---- .../server/dao/service/CustomerServiceTest.java | 6 +++--- .../server/dao/service/DeviceServiceTest.java | 6 +++--- .../server/dao/service/EntityServiceRegistryTest.java | 2 +- .../validator/CalculatedFieldDataValidatorTest.java | 4 ++-- .../validator/CalculatedFieldLinkDataValidatorTest.java | 4 ++-- 28 files changed, 61 insertions(+), 66 deletions(-) rename common/dao-api/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldService.java (87%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedField.java (97%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldConfig.java (94%) rename common/data/src/main/java/org/thingsboard/server/common/data/{calculated_field => cf}/CalculatedFieldLink.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/BaseCalculatedFieldService.java (96%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldConfigUtil.java (97%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldDao.java (89%) rename dao/src/main/java/org/thingsboard/server/dao/{calculated_field => cf}/CalculatedFieldLinkDao.java (88%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldLinkRepository.java (94%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/CalculatedFieldRepository.java (95%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldDao.java (92%) rename dao/src/main/java/org/thingsboard/server/dao/sql/{calculated_field => cf}/JpaCalculatedFieldLinkDao.java (91%) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 6d52bdb358..eb6fec88b5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -67,7 +67,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; @@ -126,7 +126,7 @@ import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index c33fb86ef8..2e9989334d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -28,23 +28,21 @@ import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java index 18d84b02ce..2362081ff9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/CalculatedFieldsDeletionTaskProcessor.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; @Component @RequiredArgsConstructor diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 52271bef83..affab6ee8b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,8 +21,8 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java similarity index 87% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b60c25e749..8171759def 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java similarity index 97% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 2482a7a19a..5c52ad0609 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java index b1d63a9396..b51258b2ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java similarity index 95% rename from common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index f24746abc0..2f176b13d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/calculated_field/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.calculated_field; +package org.thingsboard.server.common.data.cf; -import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java similarity index 96% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0b2ba43fbb..fe4422ee71 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -37,7 +37,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; -import java.util.Objects; import java.util.Optional; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java similarity index 97% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 8f9422b354..34dd885e12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java similarity index 89% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index e4ec365f46..b755d4b6c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java similarity index 88% rename from dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 7fa0285b9a..95259e75c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/calculated_field/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.calculated_field; +package org.thingsboard.server.dao.cf; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index cda9222f76..7cf7b445d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 43b4f2fc9f..98eb050d28 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +33,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 46c81e4449..73a2f9fbdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -23,7 +23,7 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +34,8 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.calculated_field.CalculatedFieldConfigUtil.toCalculatedFieldConfig; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; +import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index e50f2c902a..80e421f350 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java index 83c30d4d03..2b11f8fe42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidator.java @@ -17,9 +17,9 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index a4f035c2c9..fa1c5ce38e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index d446d59859..333057e8c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java similarity index 92% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index affd652514..3fbfe92efe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; @@ -21,11 +21,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f77baa4e42..b9ed6dc3a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/calculated_field/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.calculated_field; +package org.thingsboard.server.dao.sql.cf; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 39386e4668..ba0c73fb09 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -41,7 +41,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 32b7cbc00a..632e4b19e6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,13 +23,13 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index c021512a44..548a5bfcf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,13 +31,13 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.exception.DataValidationException; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index a9ed4d201f..16bb08350b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,8 +39,8 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -52,7 +52,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java index 0769479f48..147e69d1cb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceRegistryTest.java @@ -20,7 +20,7 @@ import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.rule.RuleChainService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index bc7990ca64..6d0eb0ad38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -19,10 +19,10 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldDao; +import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java index c9fa57a751..5ec6841150 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldLinkDataValidatorTest.java @@ -19,11 +19,11 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.common.data.calculated_field.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.calculated_field.CalculatedFieldLinkDao; +import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.UUID; From 4dbc35273ae9182024829ca558336b92fce5de6d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 13:26:48 +0200 Subject: [PATCH 018/438] added calculated field service to ctx --- .../server/actors/ActorSystemContext.java | 5 +++++ .../actors/ruleChain/DefaultTbContext.java | 6 ++++++ .../server/dao/cf/CalculatedFieldService.java | 3 +++ .../dao/cf/BaseCalculatedFieldService.java | 9 +++++++++ .../thingsboard/rule/engine/api/TbContext.java | 3 +++ .../rule/engine/util/TenantIdLoader.java | 14 ++++++++++++++ .../rule/engine/util/TenantIdLoaderTest.java | 17 +++++++++++++++++ 7 files changed, 57 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 5966716ed8..400374c731 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -62,6 +62,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.ClaimDevicesService; @@ -389,6 +390,10 @@ public class ActorSystemContext { @Getter private SlackService slackService; + @Autowired + @Getter + private CalculatedFieldService calculatedFieldService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index ec10402821..274d50a6c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -78,6 +78,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -848,6 +849,11 @@ class DefaultTbContext implements TbContext { return mainCtx.getSlackService(); } + @Override + public CalculatedFieldService getCalculatedFieldService() { + return mainCtx.getCalculatedFieldService(); + } + @Override public boolean isExternalNodeForceAck() { return mainCtx.isExternalNodeForceAck(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 8171759def..b2da1d3fe4 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; @@ -34,6 +35,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink); + CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fe4422ee71..be21c9b887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -114,6 +115,14 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } } + @Override + public CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkById, tenantId [{}], calculatedFieldLinkId [{}]", tenantId, calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java index 7dd4505f29..22c7ced237 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TbContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -357,6 +358,8 @@ public interface TbContext { SlackService getSlackService(); + CalculatedFieldService getCalculatedFieldService(); + boolean isExternalNodeForceAck(); /** diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 1e1b031259..98724b7b68 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -18,10 +18,13 @@ package org.thingsboard.rule.engine.util; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.ApiUsageStateId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceId; @@ -157,6 +160,17 @@ public class TenantIdLoader { case MOBILE_APP: tenantEntity = ctx.getMobileAppService().findMobileAppById(ctxTenantId, new MobileAppId(id)); break; + case CALCULATED_FIELD: + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, new CalculatedFieldId(id)); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = ctx.getCalculatedFieldService().findCalculatedFieldLinkById(ctxTenantId, new CalculatedFieldLinkId(id)); + if (calculatedFieldLink != null) { + tenantEntity = ctx.getCalculatedFieldService().findById(ctxTenantId, calculatedFieldLink.getCalculatedFieldId()); + } else { + tenantEntity = null; + } + break; default: throw new RuntimeException("Unexpected entity type: " + entityId.getEntityType()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index 3e0e479c80..2234e50a16 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -43,6 +43,8 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -66,6 +68,7 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; @@ -151,6 +154,8 @@ public class TenantIdLoaderTest { private DomainService domainService; @Mock private MobileAppService mobileAppService; + @Mock + private CalculatedFieldService calculatedFieldService; private TenantId tenantId; private TenantProfileId tenantProfileId; @@ -392,6 +397,18 @@ public class TenantIdLoaderTest { when(ctx.getMobileAppService()).thenReturn(mobileAppService); doReturn(mobileApp).when(mobileAppService).findMobileAppById(eq(tenantId), any()); break; + case CALCULATED_FIELD: + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedField).when(calculatedFieldService).findById(eq(tenantId), any()); + break; + case CALCULATED_FIELD_LINK: + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setTenantId(tenantId); + when(ctx.getCalculatedFieldService()).thenReturn(calculatedFieldService); + doReturn(calculatedFieldLink).when(calculatedFieldService).findCalculatedFieldLinkById(eq(tenantId), any()); + break; default: throw new RuntimeException("Unexpected originator EntityType " + entityType); } From bcef71f78d6ceb6a36703aa11660ea9a0383df06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 7 Nov 2024 14:07:58 +0200 Subject: [PATCH 019/438] added tbCalculatedField Service --- .../controller/CalculatedFieldController.java | 39 +---- .../entitiy/AbstractTbEntityService.java | 47 ++++++ .../cf/DefaultTbCalculatedFieldService.java | 149 ++++++++++++++++++ .../entitiy/cf/TbCalculatedFieldService.java | 33 ++++ .../server/dao/cf/CalculatedFieldService.java | 6 + .../dao/cf/BaseCalculatedFieldService.java | 42 ++--- .../server/dao/cf/CalculatedFieldDao.java | 2 + .../server/dao/cf/CalculatedFieldLinkDao.java | 4 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 + .../service/CalculatedFieldServiceTest.java | 10 -- 11 files changed, 270 insertions(+), 73 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 2e9989334d..4bf243034b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -27,23 +27,14 @@ import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.config.annotations.ApiOperation; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; - import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -54,10 +45,7 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j public class CalculatedFieldController extends BaseController { - private static final Set supportedEntityTypesForReferencedEntities = EnumSet.of( - EntityType.TENANT, EntityType.CUSTOMER, EntityType.ASSET, EntityType.DEVICE); - - private final CalculatedFieldService calculatedFieldService; + private final TbCalculatedFieldService tbCalculatedFieldService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; @@ -75,8 +63,7 @@ public class CalculatedFieldController extends BaseController { @RequestBody CalculatedField calculatedField) throws Exception { calculatedField.setTenantId(getTenantId()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - checkReferencedEntities(calculatedField.getConfiguration()); - return calculatedFieldService.save(calculatedField); + return tbCalculatedFieldService.save(calculatedField, getCurrentUser()); } @ApiOperation(value = "Get Calculated Field (getCalculatedFieldById)", @@ -88,7 +75,7 @@ public class CalculatedFieldController extends BaseController { public CalculatedField getCalculatedFieldById(@Parameter @PathVariable(CALCULATED_FIELD_ID) String strCalculatedFieldId) throws ThingsboardException { checkParameter(CALCULATED_FIELD_ID, strCalculatedFieldId); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedFieldId)); - CalculatedField calculatedField = calculatedFieldService.findById(getTenantId(), calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkNotNull(calculatedField); checkEntityId(calculatedField.getEntityId(), Operation.READ_CALCULATED_FIELD); return calculatedField; @@ -103,23 +90,9 @@ public class CalculatedFieldController extends BaseController { public void deleteCalculatedField(@PathVariable(CALCULATED_FIELD_ID) String strCalculatedField) throws Exception { checkParameter(CALCULATED_FIELD_ID, strCalculatedField); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(toUUID(strCalculatedField)); - TenantId tenantId = getTenantId(); - CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField calculatedField = tbCalculatedFieldService.findById(calculatedFieldId, getCurrentUser()); checkEntityId(calculatedField.getEntityId(), Operation.WRITE_CALCULATED_FIELD); - calculatedFieldService.deleteCalculatedField(getTenantId(), calculatedFieldId); - } - - private void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); - for (EntityId referencedEntityId : referencedEntityIds) { - if (!supportedEntityTypesForReferencedEntities.contains(referencedEntityId.getEntityType())) { - throw new IllegalArgumentException("Calculated fields do not support entity type '" + referencedEntityId.getEntityType() + "' for referenced entities."); - } - checkEntityId(referencedEntityId, Operation.READ); - } + tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 438cb58b91..2b19d34e57 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -25,16 +25,30 @@ import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.util.ThrowingBiFunction; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -43,6 +57,8 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import static org.thingsboard.server.dao.service.Validator.validateId; + @Slf4j public abstract class AbstractTbEntityService { @@ -71,6 +87,18 @@ public abstract class AbstractTbEntityService { @Autowired(required = false) @Lazy private EntitiesVersionControlService vcService; + @Autowired + protected AccessControlService accessControlService; + @Autowired + protected TenantService tenantService; + @Autowired + protected AssetService assetService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected AssetProfileService assetProfileService; + @Autowired + protected DeviceProfileService deviceProfileService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); @@ -120,4 +148,23 @@ public abstract class AbstractTbEntityService { return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); } } + + protected & HasTenantId, I extends EntityId> E checkEntityId(I entityId, ThrowingBiFunction findingFunction, Operation operation, SecurityUser user) throws Exception { + try { + validateId((UUIDBased) entityId, "Invalid entity id"); + E entity = findingFunction.apply(user.getTenantId(), entityId); + checkNotNull(entity, entityId.getEntityType().getNormalName() + " with id [" + entityId + "] is not found"); + return checkEntity(user, entity, operation); + } catch (Exception e) { + throw e; + } + } + + + protected & HasTenantId, I extends EntityId> E checkEntity(SecurityUser user, E entity, Operation operation) throws ThingsboardException { + checkNotNull(entity, "Entity not found"); + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), entity); + return entity; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java new file mode 100644 index 0000000000..a7c77cf4bc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.service.Validator.validateEntityId; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + + private final Map calculatedFields; + private final Map calculatedFieldLinks; + private final CalculatedFieldService calculatedFieldService; + + public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { + this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); + this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); + this.calculatedFieldService = calculatedFieldService; + } + + @Override + public void onMsg() { + + } + + @Override + public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { + ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + TenantId tenantId = calculatedField.getTenantId(); + try { + checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkReferencedEntities(calculatedField.getConfiguration(), user); + CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); + logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); + return savedCalculatedField; + } catch (ThingsboardException e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), calculatedField, actionType, user, e); + throw e; + } + } + + @Override + public CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user) { + return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); + } + + @Override + @Transactional + public void delete(CalculatedField calculatedField, SecurityUser user) { + ActionType actionType = ActionType.DELETED; + TenantId tenantId = calculatedField.getTenantId(); + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + try { + calculatedFieldService.deleteCalculatedField(tenantId, calculatedFieldId); + logEntityActionService.logEntityAction(tenantId, calculatedFieldId, calculatedField, actionType, user, calculatedFieldId.toString()); + } catch (Exception e) { + logEntityActionService.logEntityAction(tenantId, emptyId(EntityType.CALCULATED_FIELD), actionType, user, e, calculatedFieldId.toString()); + throw e; + } + } + + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { + switch (entityId.getEntityType()) { + case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); + case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); + case ASSET_PROFILE -> + Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); + case DEVICE_PROFILE -> + Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) + .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + default -> + throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); + } + } + + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() + .map(CalculatedFieldConfig.Argument::getEntityId) + .filter(Objects::nonNull) + .toList(); + for (EntityId referencedEntityId : referencedEntityIds) { + validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); + E entity = findEntity(user.getTenantId(), referencedEntityId); + checkNotNull(entity); + checkEntity(user, entity, Operation.READ); + } + + } + + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { + return (E) switch (entityId.getEntityType()) { + case TENANT -> tenantService.findTenantById((TenantId) entityId); + case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); + case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); + case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java new file mode 100644 index 0000000000..2dc1cc35b2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface TbCalculatedFieldService { + + void onMsg(); + + CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; + + CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + + void delete(CalculatedField calculatedField, SecurityUser user); + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b2da1d3fe4..7760c49d2d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -23,12 +23,16 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.List; + public interface CalculatedFieldService extends EntityDaoService { CalculatedField save(CalculatedField calculatedField); CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); @@ -37,6 +41,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index be21c9b887..ecc5aecce1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -22,19 +22,11 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -53,10 +45,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { private final CalculatedFieldDao calculatedFieldDao; private final CalculatedFieldLinkDao calculatedFieldLinkDao; - private final DeviceService deviceService; - private final AssetService assetService; - private final DeviceProfileService deviceProfileService; - private final AssetProfileService assetProfileService; private final DataValidator calculatedFieldDataValidator; private final DataValidator calculatedFieldLinkDataValidator; @@ -65,7 +53,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); - checkEntityExistence(tenantId, calculatedField.getEntityId()); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); @@ -86,6 +73,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public List findAll() { + log.trace("Executing findAll"); + return calculatedFieldDao.findAll(); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -123,6 +116,12 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public List findAllCalculatedFieldLinks() { + log.trace("Executing findAllCalculatedFieldLinks"); + return calculatedFieldLinkDao.findAll(); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -148,23 +147,6 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return EntityType.CALCULATED_FIELD; } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { - switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); - default -> - throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); - } - } - private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldLink existingLink = (calculatedField.getId() != null) ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index b755d4b6c0..bb52d0e2c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -28,6 +28,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findAll(); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 95259e75c4..c4a06c88a3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -20,8 +20,12 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.List; + public interface CalculatedFieldLinkDao extends Dao { CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 3fbfe92efe..4bf4c1ead7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -51,6 +51,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index b9ed6dc3a4..f584a8d76c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.UUID; @Slf4j @@ -44,6 +45,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll() { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 632e4b19e6..c4ad2856a9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -84,16 +84,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - @Test - public void testSaveCalculatesFieldWithNonExistingDeviceId() { - DeviceId deviceId = new DeviceId(UUID.fromString("038f8668-c9fd-4f00-8501-ce20f2f93c22")); - CalculatedField calculatedField = getCalculatedField(deviceId, deviceId); - - assertThatThrownBy(() -> calculatedFieldService.save(calculatedField)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Device with id [" + calculatedField.getEntityId().getId() + "] does not exist."); - } - @Test public void testSaveCalculatedFieldWithExistingName() { Device device = createTestDevice(); From 86dd8e725bf701f0c0fe561c3e2b6e8729235305 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 11 Nov 2024 17:29:24 +0200 Subject: [PATCH 020/438] added onCalculatedFieldAdded implementation --- .../entitiy/AbstractTbEntityService.java | 3 + .../entitiy/cf/CalculatedFieldCtx.java | 31 ++ .../entitiy/cf/CalculatedFieldState.java | 30 ++ .../cf/DefaultTbCalculatedFieldService.java | 281 ++++++++++++++++-- .../entitiy/cf/TbCalculatedFieldService.java | 8 +- .../CalculatedFieldControllerTest.java | 2 +- .../server/dao/asset/AssetService.java | 2 + .../server/dao/cf/CalculatedFieldService.java | 10 +- .../server/dao/device/DeviceService.java | 2 + .../server/dao/entity/EntityService.java | 3 + .../cf/BaseCalculatedFieldConfiguration.java | 158 ++++++++++ .../common/data/cf/CalculatedField.java | 8 +- .../data/cf/CalculatedFieldConfiguration.java | 48 +++ .../common/data/cf/CalculatedFieldLink.java | 4 +- .../cf/CalculatedFiledLinkConfiguration.java | 29 ++ .../SimpleCalculatedFieldConfiguration.java | 39 +++ common/proto/src/main/proto/queue.proto | 21 ++ .../server/dao/asset/AssetDao.java | 10 + .../server/dao/asset/BaseAssetService.java | 9 + .../dao/cf/BaseCalculatedFieldService.java | 60 ++-- .../dao/cf/CalculatedFieldConfigUtil.java | 172 +++++------ .../server/dao/cf/CalculatedFieldDao.java | 4 + .../server/dao/cf/CalculatedFieldLinkDao.java | 6 +- .../server/dao/device/DeviceDao.java | 12 + .../server/dao/device/DeviceServiceImpl.java | 9 + .../server/dao/entity/BaseEntityService.java | 5 + .../dao/model/sql/CalculatedFieldEntity.java | 18 +- .../model/sql/CalculatedFieldLinkEntity.java | 9 +- .../server/dao/sql/asset/AssetRepository.java | 10 + .../server/dao/sql/asset/JpaAssetDao.java | 11 + .../sql/cf/CalculatedFieldLinkRepository.java | 3 +- ...efaultNativeCalculatedFieldRepository.java | 146 +++++++++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 11 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 14 +- .../cf/NativeCalculatedFieldRepository.java | 29 ++ .../dao/sql/device/DeviceRepository.java | 8 + .../server/dao/sql/device/JpaDeviceDao.java | 11 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 41 files changed, 1085 insertions(+), 159 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 2b19d34e57..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; @@ -99,6 +100,8 @@ public abstract class AbstractTbEntityService { protected AssetProfileService assetProfileService; @Autowired protected DeviceProfileService deviceProfileService; + @Autowired + protected EntityService entityService; protected boolean isTestProfile() { return Set.of(this.env.getActiveProfiles()).contains("test"); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java new file mode 100644 index 0000000000..61f28304f2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@Builder +public class CalculatedFieldCtx { + + private final CalculatedFieldId calculatedFieldId; + private final EntityId entityId; + private final CalculatedFieldState state; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java new file mode 100644 index 0000000000..dc07b820ea --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class CalculatedFieldState { + + Map arguments; + String result; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index a7c77cf4bc..836f17650e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,36 +15,64 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -52,21 +80,213 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @TbCoreComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private final Map calculatedFields; - private final Map calculatedFieldLinks; private final CalculatedFieldService calculatedFieldService; + private final TbDeviceProfileCache deviceProfileCache; + private final TbAssetProfileCache assetProfileCache; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private ListeningScheduledExecutorService scheduledExecutor; - public DefaultTbCalculatedFieldService(CalculatedFieldService calculatedFieldService) { - this.calculatedFields = calculatedFieldService.findAll().stream().collect(Collectors.toMap(CalculatedField::getId, cf -> cf)); - this.calculatedFieldLinks = calculatedFieldService.findAllCalculatedFieldLinks().stream().collect(Collectors.toMap(CalculatedFieldLink::getId, cfl -> cfl)); - this.calculatedFieldService = calculatedFieldService; + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${state.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Value("10") + @Getter + private int defaultCalculatedFieldCheckIntervalInSec; + + @PostConstruct + public void init() { + // from AbstractPartitionBasedService + scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); + /// + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + // from AbstractPartitionBasedService + if (scheduledExecutor != null) { + scheduledExecutor.shutdown(); + } + /// + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { + return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); + } + + private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { + return timeseriesService.findLatest(tenantId, entityId, keys); + } + + private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { + ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); + ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); + + ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); + + return Futures.transform(combinedFuture, results -> { + List attributes = (List) results.get(0); + List timeSeries = (List) results.get(1); + + initializeState(calculatedField, attributes, timeSeries); + + return null; + }, MoreExecutors.directExecutor()); + } + + private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + String calculation = performCalculation(state.getArguments()); + + Map updatedArguments = state.getArguments(); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + Map arguments = calculatedField.getConfiguration().getArguments(); + + Map argumentValues = arguments.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) + )); + + String calculation = performCalculation(argumentValues); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + + calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, + List attributes, List timeSeries) { + String type = argument.getType(); + String value = null; + + if ("ATTRIBUTES".equals(type)) { + value = attributes.stream() + .filter(attribute -> attribute.getKey().equals(key)) + .map(AttributeKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } else if ("TIME_SERIES".equals(type)) { + value = timeSeries.stream() + .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) + .map(TsKvEntry::getValueAsString) + .findFirst() + .orElse(null); + } + + return value != null ? value : argument.getDefaultValue(); + } + + @Override + public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE: { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + case ASSET_PROFILE: { + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + for (AssetId assetId : assetIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + case DEVICE_PROFILE: { + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + for (DeviceId deviceId : deviceIds) { + for (CalculatedFieldLink link : links) { + CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field or entity was probably deleted while message was in queue; + callback.onSuccess(); + } + } catch (Exception e) { + log.trace("Failed to process calculated field add msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override - public void onMsg() { + public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + } catch (Exception e) { + log.trace("Failed to process calculated field update msg: [{}]", proto, e); + callback.onFailure(e); + } + } + @Override + public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { + try { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.remove(calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field delete msg: [{}]", proto, e); + callback.onFailure(e); + } } @Override @@ -105,28 +325,26 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + // TODO: read all states(CalculatedFieldCtx) + states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET -> Optional.ofNullable(assetService.findAssetById(tenantId, (AssetId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset with id [" + entityId.getId() + "] does not exist.")); - case DEVICE -> Optional.ofNullable(deviceService.findDeviceById(tenantId, (DeviceId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device with id [" + entityId.getId() + "] does not exist.")); - case ASSET_PROFILE -> - Optional.ofNullable(assetProfileService.findAssetProfileById(tenantId, (AssetProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Asset Profile with id [" + entityId.getId() + "] does not exist.")); - case DEVICE_PROFILE -> - Optional.ofNullable(deviceProfileService.findDeviceProfileById(tenantId, (DeviceProfileId) entityId)) - .orElseThrow(() -> new IllegalArgumentException("Device Profile with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } } - private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfig calculatedFieldConfig, SecurityUser user) throws ThingsboardException { - List referencedEntityIds = calculatedFieldConfig.getArguments().values().stream() - .map(CalculatedFieldConfig.Argument::getEntityId) - .filter(Objects::nonNull) - .toList(); + private & HasTenantId, I extends EntityId> void checkReferencedEntities(CalculatedFieldConfiguration calculatedFieldConfig, SecurityUser user) throws ThingsboardException { + List referencedEntityIds = calculatedFieldConfig.getReferencedEntities(); for (EntityId referencedEntityId : referencedEntityIds) { validateEntityId(referencedEntityId, id -> "Invalid entity id " + id); E entity = findEntity(user.getTenantId(), referencedEntityId); @@ -137,13 +355,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { - return (E) switch (entityId.getEntityType()) { - case TENANT -> tenantService.findTenantById((TenantId) entityId); - case CUSTOMER -> customerService.findCustomerById(tenantId, (CustomerId) entityId); - case ASSET -> assetService.findAssetById(tenantId, (AssetId) entityId); - case DEVICE -> deviceService.findDeviceById(tenantId, (DeviceId) entityId); + return switch (entityId.getEntityType()) { + case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } + private String performCalculation(Map argumentValues) { + return "calculation"; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 2dc1cc35b2..4f14270c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,11 +18,17 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onMsg(); + void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); + + void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); + + void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index affab6ee8b..3f505c78ef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -126,7 +126,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(null)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 7252db8097..2a9fe08827 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -63,6 +63,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); void deleteAssetsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 7760c49d2d..c12acade6c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.List; @@ -31,7 +33,9 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - List findAll(); + List findAllCalculatedFields(); + + PageData findAllCalculatedFields(PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -43,6 +47,10 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinks(); + List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + PageData findAllCalculatedFieldLinks(PageLink pageLink); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 005c740571..0848e5a1cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -75,6 +75,8 @@ public interface DeviceService extends EntityDaoService { PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); + PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type, PageLink pageLink); long countDevicesByTenantIdAndDeviceProfileIdAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 5f522121d7..fc8859eceb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.entity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -34,6 +35,8 @@ public interface EntityService { Optional fetchEntityCustomerId(TenantId tenantId, EntityId entityId); + Optional> fetchEntity(TenantId tenantId, EntityId entityId); + Optional fetchNameLabelAndCustomerDetails(TenantId tenantId, EntityId entityId); long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..9bb6d59428 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -0,0 +1,158 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@Data +public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + @JsonIgnore + private final ObjectMapper mapper = new ObjectMapper(); + + protected Map arguments; + protected SimpleCalculatedFieldConfiguration.Output output; + + public BaseCalculatedFieldConfiguration() { + } + + public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); + this.arguments = calculatedFieldConfig.getArguments(); + this.output = calculatedFieldConfig.getOutput(); + } + + @Override + public List getReferencedEntities() { + return arguments.values().stream() + .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + arguments.values().stream() + .filter(argument -> argument.getEntityId().equals(entityId)) + .forEach(argument -> { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().add(argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().add(argument.getKey()); + break; + } + }); + + return linkConfiguration; + } + + @Override + public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { + ObjectNode configNode = mapper.createObjectNode(); + + ObjectNode argumentsNode = configNode.putObject("arguments"); + arguments.forEach((key, argument) -> { + ObjectNode argumentNode = argumentsNode.putObject(key); + EntityId referencedEntityId = argument.getEntityId(); + if (referencedEntityId != null) { + argumentNode.put("entityType", referencedEntityId.getEntityType().name()); + argumentNode.put("entityId", referencedEntityId.getId().toString()); + } else { + argumentNode.put("entityType", entityType.name()); + argumentNode.put("entityId", entityId.toString()); + } + argumentNode.put("key", argument.getKey()); + argumentNode.put("type", argument.getType()); + argumentNode.put("defaultValue", argument.getDefaultValue()); + }); + + if (output != null) { + ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("type", output.getType()); + outputNode.put("expression", output.getExpression()); + } + + return configNode; + } + + @Data + public static class Argument { + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + } + + @Data + public static class Output { + private String type; + private String expression; + } + + private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { + if (config == null || !config.isObject()) { + return null; + } + + Map arguments = new HashMap<>(); + JsonNode argumentsNode = config.get("arguments"); + if (argumentsNode != null && argumentsNode.isObject()) { + argumentsNode.fields().forEachRemaining(entry -> { + String key = entry.getKey(); + JsonNode argumentNode = entry.getValue(); + Argument argument = new Argument(); + if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { + String referencedEntityType = argumentNode.get("entityType").asText(); + UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + } else { + argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + } + argument.setKey(argumentNode.get("key").asText()); + argument.setType(argumentNode.get("type").asText()); + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + arguments.put(key, argument); + }); + } + this.setArguments(arguments); + + JsonNode outputNode = config.get("output"); + if (outputNode != null) { + Output output = new Output(); + output.setType(outputNode.get("type").asText()); + output.setExpression(outputNode.get("expression").asText()); + this.setOutput(output); + } + + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 5c52ad0609..d96de37a39 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.io.Serializable; + @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -46,8 +48,8 @@ public class CalculatedField extends BaseData implements HasN private String name; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; - @Schema - private transient CalculatedFieldConfig configuration; + @Schema(implementation = SimpleCalculatedFieldConfiguration.class) + private transient CalculatedFieldConfiguration configuration; @Getter @Setter private Long version; @@ -63,7 +65,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfig configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java new file mode 100644 index 0000000000..e599b8c4d3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") +}) +public interface CalculatedFieldConfiguration { + + String getType(); + + Map getArguments(); + + List getReferencedEntities(); + + CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + + JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 2f176b13d2..922fda1f34 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFieldConfig configuration; + private transient CalculatedFiledLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldConfig configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java new file mode 100644 index 0000000000..26d867fd7f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class CalculatedFiledLinkConfiguration { + + private List attributes = new ArrayList<>(); + private List timeSeries = new ArrayList<>(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..d635e0d82e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +@Data +public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public SimpleCalculatedFieldConfiguration() { + super(); + } + + public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public String getType() { + return "SIMPLE"; + } +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 875a531566..e1d6320779 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,6 +1267,27 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } +message CalculatedFieldAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + +message CalculatedFieldDeleteMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; +} + /** TB Core to Version Control Service */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..8e40ec6d87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -103,6 +103,16 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD */ PageData findAssetInfosByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** + * Find asset ids by tenantId, assetProfileId and page link. + * + * @param tenantId the tenantId + * @param assetProfileId the assetProfileId + * @param pageLink the page link + * @return the list of asset objects + */ + PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink); + /** * Find assets by tenantId and assets Ids. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f4c68aa15b..2135792174 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -285,6 +285,15 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { + log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(assetProfileId, id -> INCORRECT_ASSET_PROFILE_ID + id); + validatePageLink(pageLink); + return assetDao.findAssetIdsByTenantIdAndAssetProfileId(tenantId.getId(), assetProfileId.getId(), pageLink); + } + @Override public ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds) { log.trace("Executing findAssetsByTenantIdAndIdsAsync, tenantId [{}], assetIds [{}]", tenantId, assetIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index ecc5aecce1..0210f6d70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -20,20 +20,24 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @@ -74,11 +78,18 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } @Override - public List findAll() { + public List findAllCalculatedFields() { log.trace("Executing findAll"); return calculatedFieldDao.findAll(); } + @Override + public PageData findAllCalculatedFields(PageLink pageLink) { + log.trace("Executing findAll, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldDao.findAll(pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); @@ -122,6 +133,19 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldLinkDao.findAll(); } + @Override + public List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksById, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); + } + + @Override + public PageData findAllCalculatedFieldLinks(PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAll(pageLink); + } + @Override public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); @@ -132,9 +156,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { return calculatedFieldDao.findAllByTenantId(tenantId).stream() .filter(calculatedField -> !referencedEntityId.equals(calculatedField.getEntityId())) .map(CalculatedField::getConfiguration) - .map(CalculatedFieldConfig::getArguments) - .flatMap(arguments -> arguments.values().stream()) - .anyMatch(argument -> referencedEntityId.equals(argument.getEntityId())); + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } @Override @@ -148,21 +171,22 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldLink existingLink = (calculatedField.getId() != null) - ? calculatedFieldLinkDao.findCalculatedFieldLinkByCalculatedFieldId(tenantId, calculatedField.getId()) - : null; - - CalculatedFieldLink updatedLink = buildCalculatedFieldLink(tenantId, calculatedField, existingLink); - saveCalculatedFieldLink(tenantId, updatedLink); + List links = buildCalculatedFieldLinks(tenantId, calculatedField); + links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField, CalculatedFieldLink existingLink) { - CalculatedFieldLink link = (existingLink != null) ? existingLink : new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(calculatedField.getEntityId()); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(calculatedField.getConfiguration()); - return link; + private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { + CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); + return cfConfig.getReferencedEntities().stream() + .map(referencedEntityId -> { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedField.getId()); + link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); + return link; + }) + .collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java index 34dd885e12..980fff6273 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java @@ -19,7 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -29,89 +31,89 @@ import java.util.UUID; public class CalculatedFieldConfigUtil { - public static CalculatedFieldConfig toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null) { - return null; - } - try { - CalculatedFieldConfig calculatedFieldConfig = new CalculatedFieldConfig(); - Map arguments = new HashMap<>(); - - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); - if (argumentNode.has("entityType") && argumentNode.has("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } - argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); - - if (argumentNode.has("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); - } - - arguments.put(key, argument); - }); - } - calculatedFieldConfig.setArguments(arguments); - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); - output.setType(outputNode.get("type").asText()); - output.setExpression(outputNode.get("expression").asText()); - calculatedFieldConfig.setOutput(output); - } - - return calculatedFieldConfig; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); - } - } - - public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfig calculatedFieldConfig, EntityType entityType, UUID entityId) { - if (calculatedFieldConfig == null) { - return null; - } - try { - ObjectNode configNode = JacksonUtil.newObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - calculatedFieldConfig.getArguments().forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); - argumentNode.put("defaultValue", argument.getDefaultValue()); - }); - - if (calculatedFieldConfig.getOutput() != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("type", calculatedFieldConfig.getOutput().getType()); - outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); - } - - return configNode; - - } catch (Exception e) { - throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); - } - } +// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { +// if (config == null) { +// return null; +// } +// try { +// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); +// Map arguments = new HashMap<>(); +// +// JsonNode argumentsNode = config.get("arguments"); +// if (argumentsNode != null && argumentsNode.isObject()) { +// argumentsNode.fields().forEachRemaining(entry -> { +// String key = entry.getKey(); +// JsonNode argumentNode = entry.getValue(); +// +// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); +// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { +// String referencedEntityType = argumentNode.get("entityType").asText(); +// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); +// } else { +// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); +// } +// argument.setKey(argumentNode.get("key").asText()); +// argument.setType(argumentNode.get("type").asText()); +// +// if (argumentNode.has("defaultValue")) { +// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); +// } +// +// arguments.put(key, argument); +// }); +// } +// calculatedFieldConfig.setArguments(arguments); +// +// JsonNode outputNode = config.get("output"); +// if (outputNode != null) { +// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); +// output.setType(outputNode.get("type").asText()); +// output.setExpression(outputNode.get("expression").asText()); +// calculatedFieldConfig.setOutput(output); +// } +// +// return calculatedFieldConfig; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); +// } +// } +// +// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { +// if (calculatedFieldConfig == null) { +// return null; +// } +// try { +// ObjectNode configNode = JacksonUtil.newObjectNode(); +// +// ObjectNode argumentsNode = configNode.putObject("arguments"); +// calculatedFieldConfig.getArguments().forEach((key, argument) -> { +// ObjectNode argumentNode = argumentsNode.putObject(key); +// EntityId referencedEntityId = argument.getEntityId(); +// if (referencedEntityId != null) { +// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); +// argumentNode.put("entityId", referencedEntityId.getId().toString()); +// } else { +// argumentNode.put("entityType", entityType.name()); +// argumentNode.put("entityId", entityId.toString()); +// } +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", argument.getType()); +// argumentNode.put("defaultValue", argument.getDefaultValue()); +// }); +// +// if (calculatedFieldConfig.getOutput() != null) { +// ObjectNode outputNode = configNode.putObject("output"); +// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); +// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); +// } +// +// return configNode; +// +// } catch (Exception e) { +// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); +// } +// } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index bb52d0e2c2..4abe02a09b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; @@ -30,6 +32,8 @@ public interface CalculatedFieldDao extends Dao { List findAll(); + PageData findAll(PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index c4a06c88a3..728e19b890 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,14 +18,18 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; import java.util.List; public interface CalculatedFieldLinkDao extends Dao { - CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); + PageData findAll(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..7c82df7703 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,7 +22,9 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -85,6 +87,16 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit */ PageData findDevicesByTenantIdAndType(UUID tenantId, String type, PageLink pageLink); + /** + * Find device ids by tenantId, type and page link. + * + * @param tenantId the tenantId + * @param deviceProfileId the deviceProfileId + * @param pageLink the page link + * @return the list of device objects + */ + PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink); + PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, OtaPackageType type, diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ac6b8330a9..ffebd5f032 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -395,6 +395,15 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + log.trace("Executing findDeviceIdsByTenantIdAndType, tenantId [{}], deviceProfileId [{}], pageLink [{}]", tenantId, deviceProfileId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(deviceProfileId, id -> INCORRECT_DEVICE_PROFILE_ID + id); + validatePageLink(pageLink); + return deviceDao.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId.getId(), deviceProfileId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..5efea9a6df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -134,6 +134,11 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return fetchAndConvert(tenantId, entityId, this::getNameLabelAndCustomerDetails); } + @Override + public Optional> fetchEntity(TenantId tenantId, EntityId entityId) { + return fetchAndConvert(tenantId, entityId, Function.identity()); + } + private Optional fetchAndConvert(TenantId tenantId, EntityId entityId, Function, T> converter) { EntityDaoService entityDaoService = entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()); Optional> entityOpt = entityDaoService.findEntity(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 98eb050d28..9cb4f08640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,6 +24,8 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -33,8 +35,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -93,7 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedFieldConfigToJson(calculatedField.getConfiguration(), entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(toCalculatedFieldConfig(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -117,4 +117,14 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 73a2f9fbdf..3dc08d6bf1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -22,8 +22,10 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -34,8 +36,6 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.calculatedFieldConfigToJson; -import static org.thingsboard.server.dao.cf.CalculatedFieldConfigUtil.toCalculatedFieldConfig; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CALCULATED_FIELD_ID; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_LINK_ENTITY_ID; @@ -65,7 +65,6 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity, Expor @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT a.id FROM AssetEntity a " + + "WHERE a.tenantId = :tenantId " + + "AND a.assetProfileId = :assetProfileId " + + "AND (:textSearch IS NULL OR ilike(a.type, CONCAT('%', :textSearch, '%')) = true) ") + Page findAssetIdsByTenantIdAndAssetProfileId(@Param("tenantId") UUID tenantId, + @Param("assetProfileId") UUID assetProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT a FROM AssetEntity a WHERE a.tenantId = :tenantId " + "AND a.customerId = :customerId AND a.type = :type " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..db6472f825 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -41,6 +41,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -159,6 +160,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, AssetInfoEntity.assetInfoColumnMap))); } + @Override + public PageData findAssetIdsByTenantIdAndAssetProfileId(UUID tenantId, UUID assetProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAssetIdsByTenantIdAndAssetProfileId( + tenantId, + assetProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(AssetId::new); + } + @Override public PageData findAssetsByTenantIdAndCustomerIdAndType(UUID tenantId, UUID customerId, String type, PageLink pageLink) { return DaoUtil.toPageData(assetRepository diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index fa1c5ce38e..61c4026cca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -18,10 +18,11 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; +import java.util.List; import java.util.UUID; public interface CalculatedFieldLinkRepository extends JpaRepository { - CalculatedFieldLinkEntity findByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..8401d99128 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -0,0 +1,146 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedFieldRepository { + + private final String CF_COUNT_QUERY = "SELECT count(id) FROM calculated_field;"; + private final String CF_QUERY = "SELECT * FROM calculated_field ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final String CFL_COUNT_QUERY = "SELECT count(id) FROM calculated_field_link;"; + private final String CFL_QUERY = "SELECT * FROM calculated_field_link ORDER BY created_time ASC LIMIT %s OFFSET %s"; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + @Override + public PageData findCalculatedFields(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CF_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CF_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + String type = (String) row.get("type"); + String name = (String) row.get("name"); + int configurationVersion = (int) row.get("configuration_version"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + long version = (long) row.get("version"); + Object externalIdObj = row.get("external_id"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setId(new CalculatedFieldId(id)); + calculatedField.setCreatedTime(createdTime); + calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedField.setType(type); + calculatedField.setName(name); + calculatedField.setConfigurationVersion(configurationVersion); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setVersion(version); + calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); + + return calculatedField; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + @Override + public PageData findCalculatedFieldLinks(Pageable pageable) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(CFL_COUNT_QUERY, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(CFL_QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(row -> { + + UUID id = (UUID) row.get("id"); + long createdTime = (long) row.get("created_time"); + UUID tenantId = (UUID) row.get("tenant_id"); + EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); + UUID entityId = (UUID) row.get("entity_id"); + UUID calculatedFieldId = (UUID) row.get("calculated_field_id"); + JsonNode configuration = JacksonUtil.valueToTree(row.get("configuration")); + + CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); + calculatedFieldLink.setId(new CalculatedFieldLinkId(id)); + calculatedFieldLink.setCreatedTime(createdTime); + calculatedFieldLink.setTenantId(new TenantId(tenantId)); + calculatedFieldLink.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + calculatedFieldLink.setCalculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + calculatedFieldLink.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFiledLinkConfiguration.class)); + + return calculatedFieldLink; + }).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } + + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + String type = config.get("type").asText(); + switch (type) { + case "SIMPLE": + return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + default: + throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); + } + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 4bf4c1ead7..1137b91947 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,16 +18,20 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -40,6 +44,7 @@ import java.util.UUID; public class JpaCalculatedFieldDao extends JpaAbstractDao implements CalculatedFieldDao { private final CalculatedFieldRepository calculatedFieldRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override public boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { @@ -56,6 +61,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated fields by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFields(DaoUtil.toPageable(pageLink)); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index f584a8d76c..a2f8f224c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -22,7 +22,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldLinkDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -39,10 +42,11 @@ import java.util.UUID; public class JpaCalculatedFieldLinkDao extends JpaAbstractDao implements CalculatedFieldLinkDao { private final CalculatedFieldLinkRepository calculatedFieldLinkRepository; + private final NativeCalculatedFieldRepository nativeCalculatedFieldRepository; @Override - public CalculatedFieldLink findCalculatedFieldLinkByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return DaoUtil.getData(calculatedFieldLinkRepository.findByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); + public List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndCalculatedFieldId(tenantId.getId(), calculatedFieldId.getId())); } @Override @@ -50,6 +54,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAll(PageLink pageLink) { + log.debug("Try to find calculated field links by pageLink [{}]", pageLink); + return nativeCalculatedFieldRepository.findCalculatedFieldLinks(DaoUtil.toPageable(pageLink)); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java new file mode 100644 index 0000000000..76fed8c311 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/NativeCalculatedFieldRepository.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.cf; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeCalculatedFieldRepository { + + PageData findCalculatedFields(Pageable pageable); + + PageData findCalculatedFieldLinks(Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..5093da78c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -81,6 +81,14 @@ public interface DeviceRepository extends JpaRepository, Exp @Param("textSearch") String textSearch, Pageable pageable); + @Query("SELECT d.id FROM DeviceEntity d WHERE d.tenantId = :tenantId " + + "AND d.deviceProfileId = :deviceProfileId " + + "AND (:textSearch IS NULL OR ilike(d.type, CONCAT('%', :textSearch, '%')) = true)") + Page findIdsByTenantIdAndDeviceProfileId(@Param("tenantId") UUID tenantId, + @Param("deviceProfileId") UUID deviceProfileId, + @Param("textSearch") String textSearch, + Pageable pageable); + @Query("SELECT d FROM DeviceEntity d WHERE d.tenantId = :tenantId " + "AND d.deviceProfileId = :deviceProfileId " + "AND d.firmwareId IS NULL") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..bf0def9fca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -173,6 +173,17 @@ public class JpaDeviceDao extends JpaAbstractDao implement DaoUtil.toPageable(pageLink))); } + @Override + public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData( + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) + .mapData(DeviceId::new); + } + @Override public PageData findDevicesByTenantIdAndTypeAndEmptyOtaPackage(UUID tenantId, UUID deviceProfileId, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index ba0c73fb09..311bf23263 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -892,7 +892,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c4ad2856a9..8e50e1e0fb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -151,7 +151,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedField.setType("Simple"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); - calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); +// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 548a5bfcf5..3f990b9dd7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -387,7 +387,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 16bb08350b..90d04d5216 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1230,7 +1230,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setOutput(output); - calculatedField.setConfiguration(config); +// calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); From 444c5bf07976df4e8ac7d57fb870a120dd673551 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:25:59 +0200 Subject: [PATCH 021/438] refactored service and test classes --- .../cf/DefaultTbCalculatedFieldService.java | 48 ++++--- .../cf/BaseCalculatedFieldConfiguration.java | 12 +- .../common/data/cf/CalculatedFieldConfig.java | 43 ------- .../data/cf/CalculatedFieldConfiguration.java | 7 +- .../common/data/cf/CalculatedFieldLink.java | 4 +- ... => CalculatedFieldLinkConfiguration.java} | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 9 +- .../dao/cf/CalculatedFieldConfigUtil.java | 119 ------------------ .../dao/model/sql/CalculatedFieldEntity.java | 3 +- .../model/sql/CalculatedFieldLinkEntity.java | 4 +- ...efaultNativeCalculatedFieldRepository.java | 9 +- .../main/resources/sql/schema-entities.sql | 4 +- .../server/dao/service/AssetServiceTest.java | 12 +- .../service/CalculatedFieldServiceTest.java | 36 +++--- .../dao/service/CustomerServiceTest.java | 12 +- .../server/dao/service/DeviceServiceTest.java | 13 +- 16 files changed, 84 insertions(+), 253 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{CalculatedFiledLinkConfiguration.java => CalculatedFieldLinkConfiguration.java} (94%) delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 836f17650e..e222259a53 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -58,8 +58,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -84,8 +82,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final TbDeviceProfileCache deviceProfileCache; - private final TbAssetProfileCache assetProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private ListeningScheduledExecutorService scheduledExecutor; @@ -215,6 +211,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return value != null ? value : argument.getDefaultValue(); } + private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { + for (T profileId : profileIds) { + for (CalculatedFieldLink link : links) { + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); + initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); + } + } + } + @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -227,33 +232,24 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE: { + case ASSET, DEVICE -> { for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); + CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); } } - case ASSET_PROFILE: { + case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - for (AssetId assetId : assetIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, assetId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, assetIds); } - case DEVICE_PROFILE: { + case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - for (DeviceId deviceId : deviceIds) { - for (CalculatedFieldLink link : links) { - CalculatedFiledLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, deviceId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + initializeForProfile(tenantId, links, cf, deviceIds); } - default: throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } } else { //Calculated field or entity was probably deleted while message was in queue; @@ -336,8 +332,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { - case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) - .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); + case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> + Optional.ofNullable(entityService.fetchEntity(tenantId, entityId)) + .orElseThrow(() -> new IllegalArgumentException(entityId.getEntityType().getNormalName() + " with id [" + entityId.getId() + "] does not exist.")); default -> throw new IllegalArgumentException("Entity type '" + entityId.getEntityType() + "' does not support calculated fields."); } @@ -357,7 +354,8 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); - default -> throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); + default -> + throw new IllegalArgumentException("Calculated fields do not support entity type '" + entityId.getEntityType() + "' for referenced entities."); }; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 9bb6d59428..3894835abc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; @@ -37,7 +38,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; - protected SimpleCalculatedFieldConfiguration.Output output; + protected Output output; public BaseCalculatedFieldConfiguration() { } @@ -45,19 +46,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); - this.output = calculatedFieldConfig.getOutput(); + this.output = calculatedFieldConfig.getOutput(); } @Override public List getReferencedEntities() { return arguments.values().stream() - .map(SimpleCalculatedFieldConfiguration.Argument::getEntityId) + .map(Argument::getEntityId) + .filter(Objects::nonNull) .collect(Collectors.toList()); } @Override - public CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId) { - CalculatedFiledLinkConfiguration linkConfiguration = new CalculatedFiledLinkConfiguration(); + public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { + CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.values().stream() .filter(argument -> argument.getEntityId().equals(entityId)) .forEach(argument -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java deleted file mode 100644 index b51258b2ec..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.Map; - -@Data -public class CalculatedFieldConfig { - - private Map arguments; - private Output output; - - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private int defaultValue; - } - - @Data - public static class Output { - private String type; - private String expression; - } - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index e599b8c4d3..deaeebcb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; @@ -35,14 +36,18 @@ import java.util.UUID; }) public interface CalculatedFieldConfiguration { + @JsonIgnore String getType(); Map getArguments(); + @JsonIgnore List getReferencedEntities(); - CalculatedFiledLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore + CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); + @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java index 922fda1f34..f9f9ee625f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLink.java @@ -37,7 +37,7 @@ public class CalculatedFieldLink extends BaseData { @Schema(description = "JSON object with the Calculated Field Id. ", accessMode = Schema.AccessMode.READ_ONLY) private CalculatedFieldId calculatedFieldId; @Schema - private transient CalculatedFiledLinkConfiguration configuration; + private transient CalculatedFieldLinkConfiguration configuration; public CalculatedFieldLink() { super(); @@ -47,7 +47,7 @@ public class CalculatedFieldLink extends BaseData { super(id); } - public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFiledLinkConfiguration configuration) { + public CalculatedFieldLink(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, CalculatedFieldLinkConfiguration configuration) { this.tenantId = tenantId; this.entityId = entityId; this.calculatedFieldId = calculatedFieldId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 26d867fd7f..02d668a67a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFiledLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; @Data -public class CalculatedFiledLinkConfiguration { +public class CalculatedFieldLinkConfiguration { private List attributes = new ArrayList<>(); private List timeSeries = new ArrayList<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0210f6d70b..db5539eb96 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -110,13 +110,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { @Override public CalculatedFieldLink saveCalculatedFieldLink(TenantId tenantId, CalculatedFieldLink calculatedFieldLink) { calculatedFieldLinkDataValidator.validate(calculatedFieldLink, CalculatedFieldLink::getTenantId); - try { - log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); - return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); - } catch (Exception e) { - checkConstraintViolation(e, "calculated_field_link_unq_key", "Calculated Field for such entity id is already exists!"); - throw e; - } + log.trace("Executing save calculated field link, [{}]", calculatedFieldLink); + return calculatedFieldLinkDao.save(tenantId, calculatedFieldLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java deleted file mode 100644 index 980fff6273..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldConfigUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.cf; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; - -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; - -public class CalculatedFieldConfigUtil { - -// public static CalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { -// if (config == null) { -// return null; -// } -// try { -// CalculatedFieldConfiguration calculatedFieldConfig = new BaseCalculatedFieldConfiguration(); -// Map arguments = new HashMap<>(); -// -// JsonNode argumentsNode = config.get("arguments"); -// if (argumentsNode != null && argumentsNode.isObject()) { -// argumentsNode.fields().forEachRemaining(entry -> { -// String key = entry.getKey(); -// JsonNode argumentNode = entry.getValue(); -// -// CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); -// if (argumentNode.has("entityType") && argumentNode.has("entityId")) { -// String referencedEntityType = argumentNode.get("entityType").asText(); -// UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); -// } else { -// argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); -// } -// argument.setKey(argumentNode.get("key").asText()); -// argument.setType(argumentNode.get("type").asText()); -// -// if (argumentNode.has("defaultValue")) { -// argument.setDefaultValue(argumentNode.get("defaultValue").asInt()); -// } -// -// arguments.put(key, argument); -// }); -// } -// calculatedFieldConfig.setArguments(arguments); -// -// JsonNode outputNode = config.get("output"); -// if (outputNode != null) { -// CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); -// output.setType(outputNode.get("type").asText()); -// output.setExpression(outputNode.get("expression").asText()); -// calculatedFieldConfig.setOutput(output); -// } -// -// return calculatedFieldConfig; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert JsonNode to CalculatedFieldConfig", e); -// } -// } -// -// public static JsonNode calculatedFieldConfigToJson(CalculatedFieldConfiguration calculatedFieldConfig, EntityType entityType, UUID entityId) { -// if (calculatedFieldConfig == null) { -// return null; -// } -// try { -// ObjectNode configNode = JacksonUtil.newObjectNode(); -// -// ObjectNode argumentsNode = configNode.putObject("arguments"); -// calculatedFieldConfig.getArguments().forEach((key, argument) -> { -// ObjectNode argumentNode = argumentsNode.putObject(key); -// EntityId referencedEntityId = argument.getEntityId(); -// if (referencedEntityId != null) { -// argumentNode.put("entityType", referencedEntityId.getEntityType().name()); -// argumentNode.put("entityId", referencedEntityId.getId().toString()); -// } else { -// argumentNode.put("entityType", entityType.name()); -// argumentNode.put("entityId", entityId.toString()); -// } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", argument.getType()); -// argumentNode.put("defaultValue", argument.getDefaultValue()); -// }); -// -// if (calculatedFieldConfig.getOutput() != null) { -// ObjectNode outputNode = configNode.putObject("output"); -// outputNode.put("type", calculatedFieldConfig.getOutput().getType()); -// outputNode.put("expression", calculatedFieldConfig.getOutput().getExpression()); -// } -// -// return configNode; -// -// } catch (Exception e) { -// throw new IllegalArgumentException("Failed to convert CalculatedFieldConfig to JsonNode", e); -// } -// } - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 9cb4f08640..725ddd7bef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -117,8 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - String type = config.get("type").asText(); + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { switch (type) { case "SIMPLE": return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 3dc08d6bf1..0ac5a9bc94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -25,7 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFiledLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -86,7 +86,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity calculatedFieldService.saveCalculatedFieldLink(tenantId, calculatedFieldLink)) - .isInstanceOf(DataValidationException.class) - .hasMessage("Calculated Field for such entity id is already exists!"); - } - private CalculatedField saveValidCalculatedField() { Device device = createTestDevice(); CalculatedField calculatedField = getCalculatedField(device.getId(), device.getId()); @@ -148,10 +140,10 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); calculatedField.setVersion(1L); return calculatedField; } @@ -160,22 +152,28 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); calculatedFieldLink.setTenantId(tenantId); calculatedFieldLink.setEntityId(calculatedField.getEntityId()); -// calculatedFieldLink.setConfiguration(calculatedField.getConfiguration()); + calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); return calculatedFieldLink; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { + CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); + calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); + return calculatedFieldLinkConfiguration; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 3f990b9dd7..0e2d9b797b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -369,25 +369,25 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(savedAsset.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 90d04d5216..33b80c0d99 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,7 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1212,25 +1212,25 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setEntityId(deviceWithCf.getId()); - CalculatedFieldConfig config = new CalculatedFieldConfig(); + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); config.setOutput(output); -// calculatedField.setConfiguration(config); + calculatedField.setConfiguration(config); CalculatedField savedCalculatedField = calculatedFieldService.save(calculatedField); @@ -1241,5 +1241,4 @@ public class DeviceServiceTest extends AbstractServiceTest { calculatedFieldService.deleteCalculatedField(tenantId, savedCalculatedField.getId()); } - } From b6b7af8003d7737c8cc5f818c85711b338329db9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 12:51:29 +0200 Subject: [PATCH 022/438] refactored tests --- .../CalculatedFieldControllerTest.java | 15 ++++++++------- .../service/CalculatedFieldServiceTest.java | 18 ------------------ 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 3f505c78ef..67e7bc64bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -22,7 +22,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfig; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -123,25 +124,25 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("Simple"); + calculatedField.setType("SIMPLE"); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); -// calculatedField.setConfiguration(getCalculatedFieldConfig(null)); + calculatedField.setConfiguration(getCalculatedFieldConfig(null)); calculatedField.setVersion(1L); return calculatedField; } - private CalculatedFieldConfig getCalculatedFieldConfig(EntityId referencedEntityId) { - CalculatedFieldConfig config = new CalculatedFieldConfig(); + private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - CalculatedFieldConfig.Argument argument = new CalculatedFieldConfig.Argument(); + SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); - CalculatedFieldConfig.Output output = new CalculatedFieldConfig.Output(); + SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index de6836900d..a94eb83fa3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,8 +25,6 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +32,6 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; -import java.util.List; import java.util.Map; import java.util.UUID; @@ -148,21 +145,6 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { return calculatedField; } - private CalculatedFieldLink getCalculatedFieldLink(CalculatedField calculatedField) { - CalculatedFieldLink calculatedFieldLink = new CalculatedFieldLink(); - calculatedFieldLink.setTenantId(tenantId); - calculatedFieldLink.setEntityId(calculatedField.getEntityId()); - calculatedFieldLink.setConfiguration(getCalculatedFieldLinkConfiguration()); - calculatedFieldLink.setCalculatedFieldId(calculatedField.getId()); - return calculatedFieldLink; - } - - private CalculatedFieldLinkConfiguration getCalculatedFieldLinkConfiguration() { - CalculatedFieldLinkConfiguration calculatedFieldLinkConfiguration = new CalculatedFieldLinkConfiguration(); - calculatedFieldLinkConfiguration.setTimeSeries(List.of("temperature")); - return calculatedFieldLinkConfiguration; - } - private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 23009c9aafd7e1dd616773621284f33abc537616 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 12 Nov 2024 17:21:03 +0200 Subject: [PATCH 023/438] refactored defaultTbCalculatedFieldService methods --- .../entitiy/AbstractTbEntityService.java | 7 + .../entitiy/cf/CalculatedFieldCtx.java | 13 +- .../cf/DefaultTbCalculatedFieldService.java | 177 +++++++----------- .../data/cf/CalculatedFieldConfiguration.java | 3 + 4 files changed, 90 insertions(+), 110 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index d2b3890c70..cef32b9065 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,18 +89,25 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired + @Lazy protected AccessControlService accessControlService; @Autowired + @Lazy protected TenantService tenantService; @Autowired + @Lazy protected AssetService assetService; @Autowired + @Lazy protected DeviceService deviceService; @Autowired + @Lazy protected AssetProfileService assetProfileService; @Autowired + @Lazy protected DeviceProfileService deviceProfileService; @Autowired + @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 61f28304f2..0a7a5c8264 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -15,17 +15,20 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @Data -@Builder public class CalculatedFieldCtx { - private final CalculatedFieldId calculatedFieldId; - private final EntityId entityId; - private final CalculatedFieldState state; + private CalculatedFieldId calculatedFieldId; + private EntityId entityId; + private CalculatedFieldState state; + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { + this.calculatedFieldId = calculatedFieldId; + this.entityId = entityId; + this.state = state; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index e222259a53..b439d8e641 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -38,7 +39,6 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -48,8 +48,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.attributes.AttributesService; @@ -62,6 +61,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -71,7 +71,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -128,98 +127,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private ListenableFuture> fetchAttributesForEntity(TenantId tenantId, EntityId entityId, List keys) { - return attributesService.find(tenantId, entityId, AttributeScope.SERVER_SCOPE, keys); - } - - private ListenableFuture> fetchTimeSeries(TenantId tenantId, EntityId entityId, List keys) { - return timeseriesService.findLatest(tenantId, entityId, keys); - } - - private ListenableFuture initializeStateFromFutures(TenantId tenantId, EntityId entityId, CalculatedField calculatedField, List attributeKeys, List timeSeriesKeys) { - ListenableFuture> attributesFuture = fetchAttributesForEntity(tenantId, entityId, attributeKeys); - ListenableFuture> timeSeriesFuture = fetchTimeSeries(tenantId, entityId, timeSeriesKeys); - - ListenableFuture> combinedFuture = Futures.allAsList(attributesFuture, timeSeriesFuture); - - return Futures.transform(combinedFuture, results -> { - List attributes = (List) results.get(0); - List timeSeries = (List) results.get(1); - - initializeState(calculatedField, attributes, timeSeries); - - return null; - }, MoreExecutors.directExecutor()); - } - - private void initializeState(CalculatedField calculatedField, List attributes, List timeSeries) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - - if (state != null) { - String calculation = performCalculation(state.getArguments()); - - Map updatedArguments = state.getArguments(); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); - } else { - // initial calculation - Map arguments = calculatedField.getConfiguration().getArguments(); - - Map argumentValues = arguments.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> resolveArgumentValue(entry.getKey(), entry.getValue(), attributes, timeSeries) - )); - - String calculation = performCalculation(argumentValues); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); - } - - calculatedFieldCtx = new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), state); - states.put(calculatedField.getId(), calculatedFieldCtx); - } - - private String resolveArgumentValue(String key, BaseCalculatedFieldConfiguration.Argument argument, - List attributes, List timeSeries) { - String type = argument.getType(); - String value = null; - - if ("ATTRIBUTES".equals(type)) { - value = attributes.stream() - .filter(attribute -> attribute.getKey().equals(key)) - .map(AttributeKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } else if ("TIME_SERIES".equals(type)) { - value = timeSeries.stream() - .filter(tsKvEntry -> tsKvEntry.getKey().equals(key)) - .map(TsKvEntry::getValueAsString) - .findFirst() - .orElse(null); - } - - return value != null ? value : argument.getDefaultValue(); - } - - private void initializeForProfile(TenantId tenantId, List links, CalculatedField cf, Iterable profileIds) { - for (T profileId : profileIds) { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, profileId, cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } - } - @Override public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { try { @@ -232,21 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - for (CalculatedFieldLink link : links) { - CalculatedFieldLinkConfiguration configuration = link.getConfiguration(); - initializeStateFromFutures(tenantId, link.getEntityId(), cf, configuration.getAttributes(), configuration.getTimeSeries()); - } - } + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, assetIds); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - initializeForProfile(tenantId, links, cf, deviceIds); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -359,7 +261,72 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private String performCalculation(Map argumentValues) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to fetch data for type: {}", argument.getType(), t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state != null) { + // calculation based on the previous data + String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); + + Map updatedArguments = new HashMap<>(state.getArguments()); + + state = CalculatedFieldState.builder() + .arguments(updatedArguments) + .result(calculation) + .build(); + } else { + // initial calculation + String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); + + state = CalculatedFieldState.builder() + .arguments(argumentValues) + .result(calculation) + .build(); + } + calculatedFieldCtx.setState(state); + states.put(calculatedField.getId(), calculatedFieldCtx); + } + + private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { + BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); return "calculation"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index deaeebcb50..cfae2e5a0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,6 +41,9 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + @JsonIgnore + BaseCalculatedFieldConfiguration.Output getOutput(); + @JsonIgnore List getReferencedEntities(); From 88e5da7a14068bb90bd2cb11b85217842446c220 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:06:12 +0200 Subject: [PATCH 024/438] changed key for states map --- .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index b439d8e641..2fe1f65015 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -90,7 +90,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); @Value("${state.initFetchPackSize:50000}") @Getter @@ -139,16 +139,16 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, callback); + case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); case ASSET_PROFILE -> { PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -180,7 +180,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); - states.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); } catch (Exception e) { log.trace("Failed to process calculated field delete msg: [{}]", proto, e); callback.onFailure(e); @@ -261,7 +261,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, TbCallback callback) { + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @@ -278,7 +278,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } }, calculatedFieldCallbackExecutor)); - updateOrInitializeState(calculatedField, argumentValues); + updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -296,8 +296,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void updateOrInitializeState(CalculatedField calculatedField, Map argumentValues) { - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(calculatedField.getId(), + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); @@ -322,7 +323,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .build(); } calculatedFieldCtx.setState(state); - states.put(calculatedField.getId(), calculatedFieldCtx); + states.put(ctxId, calculatedFieldCtx); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { From 4d8b62eb21e214cbbf2940a4ded8bac678eadf3c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 09:55:03 +0200 Subject: [PATCH 025/438] moved logic of cf add/update/delete msg to a single method --- .../cf/DefaultTbCalculatedFieldService.java | 55 +++++++++---------- .../entitiy/cf/TbCalculatedFieldService.java | 6 +- common/proto/src/main/proto/queue.proto | 19 ++----- 3 files changed, 30 insertions(+), 50 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2fe1f65015..4c83fcf7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -128,10 +128,17 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } @Override - public void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback) { + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + if (proto.getDeleted()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + if (proto.getUpdated()) { + onCalculatedFieldDelete(calculatedFieldId, callback); + } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -163,30 +170,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - @Override - public void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - } catch (Exception e) { - log.trace("Failed to process calculated field update msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - @Override - public void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback) { - try { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - } catch (Exception e) { - log.trace("Failed to process calculated field delete msg: [{}]", proto, e); - callback.onFailure(e); - } - } - @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; @@ -223,13 +206,25 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + } catch (Exception e) { + log.trace("Failed to delete calculated field.", e); + callback.onFailure(e); + } + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); // TODO: read all states(CalculatedFieldCtx) - states.keySet().removeIf(calculatedFieldId -> !calculatedFields.containsKey(calculatedFieldId)); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } private void checkEntityExistence(TenantId tenantId, EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 4f14270c3d..aa77d29702 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -24,11 +24,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldAdded(TransportProtos.CalculatedFieldAddMsgProto proto, TbCallback callback); - - void onCalculatedFieldUpdated(TransportProtos.CalculatedFieldUpdateMsgProto proto, TbCallback callback); - - void onCalculatedFieldDeleted(TransportProtos.CalculatedFieldDeleteMsgProto proto, TbCallback callback); + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index e1d6320779..b41e010a2d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1267,25 +1267,14 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldAddMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; -} - -message CalculatedFieldDeleteMsgProto { +message CalculatedFieldMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 calculatedFieldIdMSB = 3; int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; } /** From 3072861a8fa986cda6b57561a343fd3d1c6c094b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 13 Nov 2024 17:42:55 +0200 Subject: [PATCH 026/438] implemented methods for calculated field update/delete in cluster service --- application/pom.xml | 4 + .../entitiy/EntityStateSourcingListener.java | 28 +++++- .../cf/DefaultTbCalculatedFieldService.java | 11 ++- .../service/entitiy/cf/RocksDBService.java | 95 +++++++++++++++++++ .../queue/DefaultTbClusterService.java | 34 ++++++- .../queue/DefaultTbCoreConsumerService.java | 21 +++- .../server/utils/RocksDBConfig.java | 52 ++++++++++ .../src/main/resources/thingsboard.yml | 4 + .../server/cluster/TbClusterService.java | 5 + common/proto/src/main/proto/queue.proto | 1 + .../dao/cf/BaseCalculatedFieldService.java | 7 +- pom.xml | 7 ++ 12 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java diff --git a/application/pom.xml b/application/pom.xml index 17b179c767..62d2a49908 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -369,6 +369,10 @@ com.google.firebase firebase-admin + + org.rocksdb + rocksdbjni + diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 863be23e42..6c2e995b82 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -118,7 +119,11 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - default -> {} + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); + } + default -> { + } } } @@ -130,7 +135,7 @@ public class EntityStateSourcingListener { return; } EntityType entityType = entityId.getEntityType(); - if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } @@ -149,7 +154,8 @@ public class EntityStateSourcingListener { case RULE_CHAIN -> { RuleChain ruleChain = (RuleChain) event.getEntity(); if (RuleChainType.CORE.equals(ruleChain.getType())) { - Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() {}); + Set referencingRuleChainIds = JacksonUtil.fromString(event.getBody(), new TypeReference<>() { + }); if (referencingRuleChainIds != null) { referencingRuleChainIds.forEach(referencingRuleChainId -> tbClusterService.broadcastEntityStateChangeEvent(tenantId, referencingRuleChainId, ComponentLifecycleEvent.UPDATED)); @@ -177,7 +183,12 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - default -> {} + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); + } + default -> { + } } } @@ -247,6 +258,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4c83fcf7b5..06c87ff4d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AttributeScope; @@ -64,6 +65,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Random; import java.util.UUID; @@ -71,6 +73,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -83,6 +86,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final CalculatedFieldService calculatedFieldService; private final AttributesService attributesService; private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; private ListeningScheduledExecutorService scheduledExecutor; private ListeningExecutorService calculatedFieldExecutor; @@ -212,6 +216,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); + List statesToRemove = states.keySet().stream() + .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) + .collect(Collectors.toList()); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field.", e); callback.onFailure(e); @@ -223,7 +231,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - // TODO: read all states(CalculatedFieldCtx) + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -319,6 +327,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java new file mode 100644 index 0000000000..2ba3e9bb4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.springframework.stereotype.Service; +import org.thingsboard.server.utils.RocksDBConfig; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class RocksDBService { + + private final RocksDB db; + private final WriteOptions writeOptions; + + public RocksDBService(RocksDBConfig config) throws RocksDBException { + this.db = config.getDb(); + this.writeOptions = new WriteOptions().setSync(true); + } + + public void put(String key, String value) { + try { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + + public void delete(String key) { + try { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public void deleteAll(List keys) { + try (WriteBatch batch = new WriteBatch()) { + for (String key : keys) { + batch.delete(key.getBytes(StandardCharsets.UTF_8)); + } + db.write(writeOptions, batch); + } catch (RocksDBException e) { + log.error("Failed to delete data from RocksDB", e); + } + } + + public String get(String key) { + try { + byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); + return value != null ? new String(value, StandardCharsets.UTF_8) : null; + } catch (RocksDBException e) { + log.error("Failed to retrieve data from RocksDB", e); + return null; + } + } + + public Map getAll() { + Map map = new HashMap<>(); + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + String value = new String(iterator.value(), StandardCharsets.UTF_8); + map.put(key, value); + } + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } + return map; + } + +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 301f8e8838..f7c8230716 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -666,7 +668,8 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> + pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); @@ -743,4 +746,33 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { + var created = oldCalculatedField == null; + broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + + private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + builder.setAdded(added); + builder.setUpdated(updated); + builder.setDeleted(deleted); + TransportProtos.CalculatedFieldMsgProto msg = builder.build(); + pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2a64dd3388..979c419003 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -85,6 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,6 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -175,7 +178,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java new file mode 100644 index 0000000000..75a4dd138a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import jakarta.annotation.PreDestroy; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class RocksDBConfig { + + @Value("${rocksdb.db_path}") + private String dbPath; + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public RocksDB getDb() throws RocksDBException { + if (db == null) { + Options options = new Options().setCreateIfMissing(true); + db = RocksDB.open(options, dbPath); + } + return db; + } + + @PreDestroy + public void close() { + if (db != null) { + db.close(); + db = null; + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 106810a7a1..8b1aeedec8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +rocksdb: + # Rocksdb path + db_path: "${ROCKS_DB_PATH:}" + # Actor system parameters actors: system: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5112e93da7..f173005107 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -114,4 +115,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); + + void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b41e010a2d..95e4fa8601 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1512,6 +1512,7 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; + CalculatedFieldMsgProto calculatedFieldMsg = 53; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index db5539eb96..8603dadcc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -29,20 +29,21 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.entity.AbstractEntityService.checkConstraintViolation; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @Service("CalculatedFieldDaoService") @Slf4j @RequiredArgsConstructor -public class BaseCalculatedFieldService implements CalculatedFieldService { +public class BaseCalculatedFieldService extends AbstractEntityService implements CalculatedFieldService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; public static final String INCORRECT_CALCULATED_FIELD_ID = "Incorrect calculatedFieldId "; @@ -60,6 +61,8 @@ public class BaseCalculatedFieldService implements CalculatedFieldService { log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) + .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, diff --git a/pom.xml b/pom.xml index a1d5f26ecf..9b835516fc 100755 --- a/pom.xml +++ b/pom.xml @@ -166,6 +166,8 @@ 1.6.1 2.19.0 9.2.0 + + 9.4.0 @@ -2272,6 +2274,11 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From f75eaf8226cff33ad0d8f935f1750394a3475375 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 10:23:57 +0200 Subject: [PATCH 027/438] added rocksdb path --- .../entitiy/AbstractTbEntityService.java | 7 ------- .../server/utils/RocksDBConfig.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 21 +++++++++++++++++++ .../DefaultTbAlarmCommentServiceTest.java | 21 +++++++++++++++++++ 5 files changed, 44 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index cef32b9065..d2b3890c70 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -89,25 +89,18 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; @Autowired - @Lazy protected AccessControlService accessControlService; @Autowired - @Lazy protected TenantService tenantService; @Autowired - @Lazy protected AssetService assetService; @Autowired - @Lazy protected DeviceService deviceService; @Autowired - @Lazy protected AssetProfileService assetProfileService; @Autowired - @Lazy protected DeviceProfileService deviceProfileService; @Autowired - @Lazy protected EntityService entityService; protected boolean isTestProfile() { diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java index 75a4dd138a..9c3c02f472 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java @@ -25,7 +25,7 @@ import org.springframework.stereotype.Component; @Component public class RocksDBConfig { - @Value("${rocksdb.db_path}") + @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") private String dbPath; private RocksDB db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8b1aeedec8..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,7 +426,7 @@ sql: rocksdb: # Rocksdb path - db_path: "${ROCKS_DB_PATH:}" + db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" # Actor system parameters actors: diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 60a0ac3bf6..bc1726ff41 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -38,10 +38,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; @@ -81,6 +88,20 @@ public class DefaultTbAlarmServiceTest { protected TbClusterService tbClusterService; @MockBean private EntitiesVersionControlService vcService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmService service; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java index b36ca51416..71c295edc9 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarmComment/DefaultTbAlarmCommentServiceTest.java @@ -35,10 +35,17 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.entitiy.TbLogEntityActionService; import org.thingsboard.server.service.entitiy.alarm.DefaultTbAlarmCommentService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import java.util.UUID; @@ -72,6 +79,20 @@ public class DefaultTbAlarmCommentServiceTest { protected CustomerService customerService; @MockBean protected TbClusterService tbClusterService; + @MockBean + private AccessControlService accessControlService; + @MockBean + private TenantService tenantService; + @MockBean + private AssetService assetService; + @MockBean + private DeviceService deviceService; + @MockBean + private AssetProfileService assetProfileService; + @MockBean + private DeviceProfileService deviceProfileService; + @MockBean + private EntityService entityService; @SpyBean DefaultTbAlarmCommentService service; From 8079ef66efc3be7f54305758390c296cd6848356 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 14 Nov 2024 14:44:40 +0200 Subject: [PATCH 028/438] added logs --- .../entitiy/cf/CalculatedFieldCtx.java | 3 ++ .../entitiy/cf/CalculatedFieldState.java | 1 + .../cf/DefaultTbCalculatedFieldService.java | 29 ++++++++++++------- .../data/cf/CalculatedFieldConfiguration.java | 1 - .../dao/model/sql/CalculatedFieldEntity.java | 8 ++--- .../model/sql/CalculatedFieldLinkEntity.java | 4 +-- ...efaultNativeCalculatedFieldRepository.java | 4 +-- 7 files changed, 31 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 0a7a5c8264..021a9bbc83 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -26,6 +26,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldState state; + public CalculatedFieldCtx() { + } + public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { this.calculatedFieldId = calculatedFieldId; this.entityId = entityId; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index dc07b820ea..f84f4a57c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -24,6 +24,7 @@ import java.util.Map; @Builder public class CalculatedFieldState { + // TODO: use value object(TsKv) instead of string Map arguments; String result; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 06c87ff4d1..51ff540a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -67,7 +67,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Random; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -113,7 +112,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, new Random().nextInt(defaultCalculatedFieldCheckIntervalInSec), defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); + scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); } @PreDestroy @@ -136,11 +135,15 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + //TODO: improve the check. Maybe it was renamed or just the name of the result changed. onCalculatedFieldDelete(calculatedFieldId, callback); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); @@ -150,13 +153,18 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); switch (entityId.getEntityType()) { - case ASSET, DEVICE -> initializeStateForEntity(tenantId, cf, entityId, callback); + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); } case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); @@ -164,12 +172,14 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { - //Calculated field or entity was probably deleted while message was in queue; + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } } catch (Exception e) { - log.trace("Failed to process calculated field add msg: [{}]", proto, e); + log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); } } @@ -210,7 +220,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { calculatedFieldLinks.remove(calculatedFieldId); @@ -221,7 +230,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp .collect(Collectors.toList()); rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { - log.trace("Failed to delete calculated field.", e); + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); } } @@ -231,7 +240,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.convertValue(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); } @@ -276,7 +285,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp @Override public void onFailure(Throwable t) { - log.warn("Failed to fetch data for type: {}", argument.getType(), t); + log.warn("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); @@ -327,7 +336,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } calculatedFieldCtx.setState(state); states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.toString(calculatedFieldCtx))); + rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index cfae2e5a0e..7e6a77f006 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -41,7 +41,6 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - @JsonIgnore BaseCalculatedFieldConfiguration.Output getOutput(); @JsonIgnore diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 725ddd7bef..efb262477b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -56,7 +56,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem private UUID tenantId; @Column(name = CALCULATED_FIELD_ENTITY_TYPE) - private EntityType entityType; + private String entityType; @Column(name = CALCULATED_FIELD_ENTITY_ID) private UUID entityId; @@ -88,12 +88,12 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.setUuid(calculatedField.getUuidId()); this.createdTime = calculatedField.getCreatedTime(); this.tenantId = calculatedField.getTenantId().getId(); - this.entityType = calculatedField.getEntityId().getEntityType(); + this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); this.type = calculatedField.getType(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(entityType, entityId); + this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -109,7 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, entityType, entityId)); + calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java index 0ac5a9bc94..b4d6677be4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldLinkEntity.java @@ -53,7 +53,7 @@ public class CalculatedFieldLinkEntity extends BaseSqlEntity Date: Thu, 14 Nov 2024 17:19:14 +0200 Subject: [PATCH 029/438] added logic to handle cf update msg --- .../entitiy/cf/CalculatedFieldState.java | 24 ++++--- .../cf/DefaultTbCalculatedFieldService.java | 67 ++++++++++++------- .../cf/SimpleCalculatedFieldState.java | 49 ++++++++++++++ .../CalculatedFieldControllerTest.java | 3 +- .../cf/BaseCalculatedFieldConfiguration.java | 1 + .../common/data/cf/CalculatedField.java | 12 ++-- .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../common/data/cf/CalculatedFieldType.java | 22 ++++++ .../SimpleCalculatedFieldConfiguration.java | 4 +- .../dao/model/sql/CalculatedFieldEntity.java | 9 +-- ...efaultNativeCalculatedFieldRepository.java | 7 +- .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 15 files changed, 160 insertions(+), 52 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index f84f4a57c2..9fb6a009fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,17 +15,25 @@ */ package org.thingsboard.server.service.entitiy.cf; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.Map; -@Data -@Builder -public class CalculatedFieldState { +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") +}) +public interface CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - Map arguments; - String result; + CalculatedFieldType getType(); + + void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 51ff540a64..49c64a1262 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -141,12 +142,11 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - //TODO: improve the check. Maybe it was renamed or just the name of the result changed. - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldUpdate(cf, callback); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); @@ -235,6 +235,31 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + } + } + + private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + } + private void fetchCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); @@ -309,39 +334,33 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = calculatedField.getId().getId() + "_" + entityId.getId(); + String ctxId = generateCtxId(calculatedField.getId(), entityId); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - // calculation based on the previous data - String calculation = performCalculation(state.getArguments(), calculatedField.getConfiguration()); - - Map updatedArguments = new HashMap<>(state.getArguments()); - - state = CalculatedFieldState.builder() - .arguments(updatedArguments) - .result(calculation) - .build(); + state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); } else { - // initial calculation - String calculation = performCalculation(argumentValues, calculatedField.getConfiguration()); - - state = CalculatedFieldState.builder() - .arguments(argumentValues) - .result(calculation) - .build(); + CalculatedFieldState newState = createStateByType(calculatedField.getType()); + newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); } calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); } - private String performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration) { - BaseCalculatedFieldConfiguration.Output output = calculatedFieldConfiguration.getOutput(); - return "calculation"; + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + default -> + throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); + }; + } + + private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { + return calculatedFieldId.getId() + "_" + entityId.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java new file mode 100644 index 0000000000..8a90893262 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class SimpleCalculatedFieldState implements CalculatedFieldState { + + // TODO: use value object(TsKv) instead of string + Map arguments = new HashMap<>(); + String result; + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; + } + + @Override + public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { + if (initialCalculation) { + // todo: perform initial calculation + this.arguments = argumentValues; + } else { + // todo: perform calculation based on previous data + this.arguments.putAll(argumentValues); + } + this.result = "result"; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 67e7bc64bd..ebda2cbf82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -124,7 +125,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedField getCalculatedField(DeviceId deviceId) { CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(deviceId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(null)); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 3894835abc..4575e414ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -115,6 +115,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Data public static class Output { + private String name; private String type; private String expression; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index d96de37a39..ceb1222fe2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -20,15 +20,17 @@ import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -import java.io.Serializable; - @Schema @Data @EqualsAndHashCode(callSuper = true) @@ -41,7 +43,7 @@ public class CalculatedField extends BaseData implements HasN @NoXss @Length(fieldName = "type") - private String type; + private CalculatedFieldType type; @NoXss @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") @@ -65,7 +67,7 @@ public class CalculatedField extends BaseData implements HasN super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, String type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { + public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version, CalculatedFieldId externalId) { this.tenantId = tenantId; this.entityId = entityId; this.type = type; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index 7e6a77f006..f733c35310 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -37,7 +37,7 @@ import java.util.UUID; public interface CalculatedFieldConfiguration { @JsonIgnore - String getType(); + CalculatedFieldType getType(); Map getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java new file mode 100644 index 0000000000..89173b35b9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +public enum CalculatedFieldType { + + SIMPLE, SCRIPT + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java index d635e0d82e..327f9cdc75 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java @@ -33,7 +33,7 @@ public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfi } @Override - public String getType() { - return "SIMPLE"; + public CalculatedFieldType getType() { + return CalculatedFieldType.SIMPLE; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index efb262477b..3c45a81cb5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +91,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.tenantId = calculatedField.getTenantId().getId(); this.entityType = calculatedField.getEntityId().getEntityType().name(); this.entityId = calculatedField.getEntityId().getId(); - this.type = calculatedField.getType(); + this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); @@ -106,7 +107,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setCreatedTime(createdTime); calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - calculatedField.setType(type); + calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); @@ -118,8 +119,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case "SIMPLE": + switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 73f334e8f1..417a468b2c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -73,7 +74,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF UUID tenantId = (UUID) row.get("tenant_id"); EntityType entityType = EntityType.valueOf((String) row.get("entity_type")); UUID entityId = (UUID) row.get("entity_id"); - String type = (String) row.get("type"); + CalculatedFieldType type = CalculatedFieldType.valueOf((String) row.get("type")); String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); @@ -133,9 +134,9 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(String type, JsonNode config, EntityType entityType, UUID entityId) { + private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { switch (type) { - case "SIMPLE": + case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 432d559598..43b51ce2ac 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -874,7 +875,7 @@ public class AssetServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAssetWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a94eb83fa3..d5d025a46d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -25,6 +25,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -137,7 +138,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setEntityId(entityId); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 0e2d9b797b..1c5e0d8f49 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -369,7 +370,7 @@ public class CustomerServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(savedAsset.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 33b80c0d99..8f1f16e631 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1212,7 +1213,7 @@ public class DeviceServiceTest extends AbstractServiceTest { CalculatedField calculatedField = new CalculatedField(); calculatedField.setTenantId(tenantId); calculatedField.setName("Test CF"); - calculatedField.setType("SIMPLE"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setEntityId(deviceWithCf.getId()); SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); From 7c433d47e2db2b291ebd99d2047f3c2e10887178 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 08:46:35 +0200 Subject: [PATCH 030/438] changed type value in cf data validator test --- .../service/validator/CalculatedFieldDataValidatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java index 6d0eb0ad38..10df39e04b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidatorTest.java @@ -20,6 +20,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.cf.CalculatedFieldDao; @@ -44,7 +45,7 @@ public class CalculatedFieldDataValidatorTest { @Test public void testUpdateNonExistingCalculatedField() { CalculatedField calculatedField = new CalculatedField(CALCULATED_FIELD_ID); - calculatedField.setType("Simple"); + calculatedField.setType(CalculatedFieldType.SIMPLE); calculatedField.setName("Test"); given(calculatedFieldDao.findById(TENANT_ID, CALCULATED_FIELD_ID.getId())).willReturn(null); From 2fc32ee232c37cb9201e18adbebcc4d5e5ab7b24 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 12:07:41 +0200 Subject: [PATCH 031/438] removed unnecessary methods --- .../entitiy/cf/CalculatedFieldState.java | 2 + .../cf/DefaultTbCalculatedFieldService.java | 24 +++++--- .../src/main/resources/thingsboard.yml | 4 ++ .../server/dao/cf/CalculatedFieldService.java | 9 ++- .../dao/cf/BaseCalculatedFieldService.java | 55 ++++++++++++++++--- .../server/dao/cf/CalculatedFieldDao.java | 2 - .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 7 --- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 7 ++- 9 files changed, 84 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 9fb6a009fe..221c44b94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; @@ -32,6 +33,7 @@ import java.util.Map; }) public interface CalculatedFieldState { + @JsonIgnore CalculatedFieldType getType(); void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 49c64a1262..f2f16de844 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -96,7 +96,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - @Value("${state.initFetchPackSize:50000}") + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -145,7 +145,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldUpdate(cf, callback); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } } List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { @@ -172,12 +175,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); } - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } else { //Calculated field was probably deleted while message was in queue; log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); callback.onSuccess(); } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to process calculated field msg: [{}]", proto, e); callback.onFailure(e); @@ -235,17 +239,20 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - if (hasSignificantChanged(oldCalculatedField, newCalculatedField)) { + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); callback.onSuccess(); + shouldReinit = false; } + return shouldReinit; } - private boolean hasSignificantChanged(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; } @@ -255,9 +262,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputNameChanged = !oldConfig.getOutput().getName().equals(newConfig.getOutput().getName()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputNameChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; } private void fetchCalculatedFields() { @@ -344,6 +351,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } else { CalculatedFieldState newState = createStateByType(calculatedField.getType()); newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); + state = newState; } calculatedFieldCtx.setState(state); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97405f35d3..699d2012cc 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,6 +424,10 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls +# Calculated Field parameters +calculatedField: + initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" + rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index c12acade6c..4bb67f9f0d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -33,6 +34,8 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedField findById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); @@ -45,13 +48,15 @@ public interface CalculatedFieldService extends EntityDaoService { CalculatedFieldLink findCalculatedFieldLinkById(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId); + List findAllCalculatedFieldLinks(); List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); - PageData findAllCalculatedFieldLinks(PageLink pageLink); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); - boolean existsByEntityId(TenantId tenantId, EntityId entityId); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 8603dadcc9..fd669f7865 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -30,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; @@ -55,14 +58,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public CalculatedField save(CalculatedField calculatedField) { - calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); + CalculatedField oldCalculatedField = calculatedFieldDataValidator.validate(calculatedField, CalculatedField::getTenantId); try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) - .entity(savedCalculatedField).created(calculatedField.getId() == null).build()); + .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; } catch (Exception e) { checkConstraintViolation(e, @@ -80,6 +83,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findById(tenantId, calculatedFieldId.getId()); } + @Override + public ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findCalculatedFieldByIdAsync [{}]", calculatedFieldId); + validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); + return calculatedFieldDao.findByIdAsync(tenantId, calculatedFieldId.getId()); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -95,10 +105,28 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedFieldId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(calculatedFieldId, id -> INCORRECT_CALCULATED_FIELD_ID + id); - calculatedFieldDao.removeById(tenantId, calculatedFieldId.getId()); + deleteEntity(tenantId, calculatedFieldId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CalculatedField calculatedField = calculatedFieldDao.findById(tenantId, id.getId()); + if (calculatedField == null) { + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent calculated field."); + } + } + deleteCalculatedField(tenantId, calculatedField); + } + + private void deleteCalculatedField(TenantId tenantId, CalculatedField calculatedField) { + log.trace("Executing deleteCalculatedField, tenantId [{}], calculatedFieldId [{}]", tenantId, calculatedField.getId()); + calculatedFieldDao.removeById(tenantId, calculatedField.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(calculatedField.getId()).entity(calculatedField).build()); } @Override @@ -125,6 +153,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findById(tenantId, calculatedFieldLinkId.getId()); } + @Override + public ListenableFuture findCalculatedFieldLinkByIdAsync(TenantId tenantId, CalculatedFieldLinkId calculatedFieldLinkId) { + log.trace("Executing findCalculatedFieldLinkByIdAsync [{}]", calculatedFieldLinkId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validateId(calculatedFieldLinkId, id -> "Incorrect calculatedFieldLinkId " + id); + return calculatedFieldLinkDao.findByIdAsync(tenantId, calculatedFieldLinkId.getId()); + } + @Override public List findAllCalculatedFieldLinks() { log.trace("Executing findAllCalculatedFieldLinks"); @@ -137,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldIdAsync(tenantId, calculatedFieldId); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); @@ -144,11 +186,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findAll(pageLink); } - @Override - public boolean existsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldDao.existsByTenantIdAndEntityId(tenantId, entityId); - } - @Override public boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId) { return calculatedFieldDao.findAllByTenantId(tenantId).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 4abe02a09b..d1c9fd86bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -26,8 +26,6 @@ import java.util.List; public interface CalculatedFieldDao extends Dao { - boolean existsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); - List findAllByTenantId(TenantId tenantId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 728e19b890..34f2129bd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.cf; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 1137b91947..bdc701070d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sql.cf; import jakarta.transaction.Transactional; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; @@ -31,7 +30,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.List; @@ -46,11 +44,6 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId) { return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantId(tenantId.getId())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index a2f8f224c1..417b529dc9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.cf; +import com.google.common.util.concurrent.ListenableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; @@ -22,7 +23,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +49,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); From 24281b48b5f3b720a5b963322c474a9346a97f77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 18 Nov 2024 17:11:09 +0200 Subject: [PATCH 032/438] removed properrty from thingsboard.yml --- application/src/main/resources/thingsboard.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 699d2012cc..97405f35d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -424,10 +424,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -# Calculated Field parameters -calculatedField: - initFetchPackSize: "${INIT_FETCH_PACK_SIZE:50000}" - rocksdb: # Rocksdb path db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" From c39a373038c300a306dc2cf87bcfeebd898e4073 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 19 Nov 2024 17:21:25 +0200 Subject: [PATCH 033/438] added calculated field execution service --- .../cf/CalculatedFieldExecutionService.java | 25 ++ ...efaultCalculatedFieldExecutionService.java | 309 ++++++++++++++++++ .../entitiy/EntityStateSourcingListener.java | 16 +- .../entitiy/cf/CalculatedFieldCtx.java | 9 +- .../entitiy/cf/CalculatedFieldCtxId.java | 21 ++ .../entitiy/cf/CalculatedFieldResult.java | 46 +++ .../entitiy/cf/CalculatedFieldState.java | 9 +- .../cf/DefaultTbCalculatedFieldService.java | 258 --------------- .../service/entitiy/cf/RocksDBService.java | 2 + .../cf/ScriptCalculatedFieldState.java | 48 +++ .../cf/SimpleCalculatedFieldState.java | 37 ++- .../entitiy/cf/TbCalculatedFieldService.java | 4 - .../queue/DefaultTbCoreConsumerService.java | 10 +- .../CalculatedFieldControllerTest.java | 3 +- .../server/dao/cf/CalculatedFieldService.java | 2 + .../server/common/data/cf/Argument.java | 32 ++ .../cf/BaseCalculatedFieldConfiguration.java | 8 - .../data/cf/CalculatedFieldConfiguration.java | 2 +- .../dao/cf/BaseCalculatedFieldService.java | 5 + .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 + .../server/dao/service/AssetServiceTest.java | 3 +- .../service/CalculatedFieldServiceTest.java | 3 +- .../dao/service/CustomerServiceTest.java | 3 +- .../server/dao/service/DeviceServiceTest.java | 3 +- 25 files changed, 566 insertions(+), 299 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java create mode 100644 application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java new file mode 100644 index 0000000000..6daa88d771 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos; + +public interface CalculatedFieldExecutionService { + + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java new file mode 100644 index 0000000000..c0eeec02bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -0,0 +1,309 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; +import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.RocksDBService; +import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; +import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.partition.AbstractPartitionBasedService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@TbCoreComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + private final AttributesService attributesService; + private final TimeseriesService timeseriesService; + private final RocksDBService rocksDBService; + + private ListeningExecutorService calculatedFieldExecutor; + private ListeningExecutorService calculatedFieldCallbackExecutor; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @PostConstruct + public void init() { + super.init(); + calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); + calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( + Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + @PreDestroy + public void stop() { + if (calculatedFieldExecutor != null) { + calculatedFieldExecutor.shutdownNow(); + } + if (calculatedFieldCallbackExecutor != null) { + calculatedFieldCallbackExecutor.shutdownNow(); + } + } + + @Override + protected String getServiceName() { + return "Calculated Field Execution"; + } + + @Override + protected String getSchedulerExecutorName() { + return "calculated-field-scheduled"; + } + + @Override + protected Map>> onAddedPartitions(Set addedPartitions) { + // TODO: implementation for cluster mode + return Map.of(); + } + + @Override + protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { + // TODO: implementation for cluster mode + } + + @Override + public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + if (proto.getDeleted()) { + log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + onCalculatedFieldDelete(calculatedFieldId, callback); + callback.onSuccess(); + } + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (proto.getUpdated()) { + log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); + if (!shouldReinit) { + return; + } + } + List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cf != null) { + EntityId entityId = cf.getEntityId(); + calculatedFields.put(calculatedFieldId, cf); + calculatedFieldLinks.put(calculatedFieldId, links); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntity(tenantId, cf, entityId, callback); + } + case ASSET_PROFILE -> { + log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); + PageDataIterable assetIds = new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); + assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + } + case DEVICE_PROFILE -> { + log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); + PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); + deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + } + default -> + throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); + } + } else { + //Calculated field was probably deleted while message was in queue; + log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); + callback.onSuccess(); + } + callback.onSuccess(); + log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to process calculated field msg: [{}]", proto, e); + callback.onFailure(e); + } + } + + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { + onCalculatedFieldDelete(newCalculatedField.getId(), callback); + } else { + calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + calculatedFieldLinks.remove(calculatedFieldId); + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + } + + private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + Map arguments = calculatedField.getConfiguration().getArguments(); + Map argumentValues = new HashMap<>(); + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + @Override + public void onSuccess(Optional result) { + String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); + argumentValues.put(key, value); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to initialize state for entity: [{}]", entityId, t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor)); + + updateOrInitializeState(calculatedField, entityId, argumentValues); + + } + + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + return switch (argument.getType()) { + case "ATTRIBUTES" -> Futures.transform( + attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + case "TIME_SERIES" -> Futures.transform( + timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), + result -> result.map(entry -> (KvEntry) entry), + MoreExecutors.directExecutor()); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); + CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + + CalculatedFieldState state = calculatedFieldCtx.getState(); + + if (state == null) { + state = createStateByType(calculatedField.getType()); + } + state.initState(argumentValues); + calculatedFieldCtx.setState(state); + states.put(ctxId, calculatedFieldCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + + state.performCalculation(calculatedField.getConfiguration()); + } + + private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { + return switch (calculatedFieldType) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 6c2e995b82..1b541bcd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -65,6 +67,8 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; + private final CalculatedFieldService calculatedFieldService; + private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -102,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(event.getEntity(), event.getOldEntity()); + onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -241,11 +245,19 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(Object entity, Object oldEntity) { + private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; + // TODO: move verification of device type to cluster service + if (!oldDevice.getType().equals(device.getType())) { + DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); + if (cfExistsByProfile) { + // TODO: send device type updated msg to core + } + } } tbClusterService.onDeviceUpdated(device, oldDevice); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java index 021a9bbc83..8a5e4cdf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java @@ -22,16 +22,15 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class CalculatedFieldCtx { - private CalculatedFieldId calculatedFieldId; - private EntityId entityId; + private CalculatedFieldCtxId id; private CalculatedFieldState state; public CalculatedFieldCtx() { } - public CalculatedFieldCtx(CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState state) { - this.calculatedFieldId = calculatedFieldId; - this.entityId = entityId; + public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + this.id = id; this.state = state; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java new file mode 100644 index 0000000000..3dc0dead36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import java.util.UUID; + +public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java new file mode 100644 index 0000000000..adb8b70e7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class CalculatedFieldResult { + + private String name; + private String type; + private AttributeScope scope; + private String value; + + public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "ATTRIBUTES"; + result.scope = scope; + result.value = value; + return result; + } + + public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.name = name; + result.type = "TIME_SERIES"; + result.value = value; + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java index 221c44b94c..9997df947f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.entitiy.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -36,6 +37,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation); + default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { + return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + } + + void initState(Map argumentValues); + + CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index f2f16de844..0cc644606f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -15,65 +15,28 @@ */ package org.thingsboard.server.service.entitiy.cf; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.ListeningScheduledExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -84,109 +47,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { private final CalculatedFieldService calculatedFieldService; - private final AttributesService attributesService; - private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; - private ListeningScheduledExecutorService scheduledExecutor; - - private ListeningExecutorService calculatedFieldExecutor; - private ListeningExecutorService calculatedFieldCallbackExecutor; - - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - @Value("${calculatedField.initFetchPackSize:50000}") - @Getter - private int initFetchPackSize; - - @Value("10") - @Getter - private int defaultCalculatedFieldCheckIntervalInSec; - - @PostConstruct - public void init() { - // from AbstractPartitionBasedService - scheduledExecutor = MoreExecutors.listeningDecorator(Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("calculated-field-scheduled"))); - /// - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); - calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.scheduleWithFixedDelay(this::fetchCalculatedFields, 0, defaultCalculatedFieldCheckIntervalInSec, TimeUnit.SECONDS); - } - - @PreDestroy - public void stop() { - // from AbstractPartitionBasedService - if (scheduledExecutor != null) { - scheduledExecutor.shutdown(); - } - /// - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } - if (calculatedFieldCallbackExecutor != null) { - calculatedFieldCallbackExecutor.shutdownNow(); - } - } - - @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cf != null) { - EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); - } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } @Override public CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException { @@ -224,58 +84,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - calculatedFieldLinks.remove(calculatedFieldId); - calculatedFields.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.startsWith(calculatedFieldId.getId().toString())); - List statesToRemove = states.keySet().stream() - .filter(key -> key.startsWith(calculatedFieldId.getId().toString())) - .collect(Collectors.toList()); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); - } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(ctxId, JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.startsWith(id.toString()))); - } - private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> @@ -305,70 +113,4 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp }; } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { - @Override - public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); - } - - @Override - public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - - } - - private ListenableFuture> fetchArgumentValue(TenantId tenantId, BaseCalculatedFieldConfiguration.Argument argument) { - return switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), - MoreExecutors.directExecutor()); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - String ctxId = generateCtxId(calculatedField.getId(), entityId); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, - ctx -> new CalculatedFieldCtx(calculatedField.getId(), calculatedField.getEntityId(), null)); - - CalculatedFieldState state = calculatedFieldCtx.getState(); - if (state != null) { - state.performCalculation(argumentValues, calculatedField.getConfiguration(), false); - } else { - CalculatedFieldState newState = createStateByType(calculatedField.getType()); - newState.performCalculation(argumentValues, calculatedField.getConfiguration(), true); - state = newState; - } - calculatedFieldCtx.setState(state); - - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(ctxId, Objects.requireNonNull(JacksonUtil.writeValueAsString(calculatedFieldCtx))); - } - - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - default -> - throw new IllegalArgumentException("Invalid calculated field type '" + calculatedFieldType + "'."); - }; - } - - private String generateCtxId(CalculatedFieldId calculatedFieldId, EntityId entityId) { - return calculatedFieldId.getId() + "_" + entityId.getId(); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java index 2ba3e9bb4f..2cf7aec18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.utils.RocksDBConfig; @@ -31,6 +32,7 @@ import java.util.Map; @Service @Slf4j +@ConditionalOnExpression("'${service.type:null}'=='monolith'") public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java new file mode 100644 index 0000000000..52435643cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.cf; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class ScriptCalculatedFieldState implements CalculatedFieldState { + + private TbelInvokeService tbelInvokeService; + + private Map arguments = new HashMap<>(); + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } + + @Override + public void initState(Map argumentValues) { + + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java index 8a90893262..c004f9dd65 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.entitiy.cf; import lombok.Data; +import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -26,8 +28,8 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - Map arguments = new HashMap<>(); - String result; + private Map arguments = new HashMap<>(); + private String outputResult; @Override public CalculatedFieldType getType() { @@ -35,15 +37,30 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void performCalculation(Map argumentValues, CalculatedFieldConfiguration calculatedFieldConfiguration, boolean initialCalculation) { - if (initialCalculation) { - // todo: perform initial calculation - this.arguments = argumentValues; - } else { - // todo: perform calculation based on previous data - this.arguments.putAll(argumentValues); + public void initState(Map argumentValues) { + this.arguments = argumentValues; + } + + @Override + public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + if (isValid(arguments, calculatedFieldConfiguration)) { + String expression = calculatedFieldConfiguration.getOutput().getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(arguments.keySet()) + .build(); + customExpression.set(expr); + } + Map variables = new HashMap<>(); + arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + expr.setVariables(variables); + double result = expr.evaluate(); + this.outputResult = Double.toString(result); } - this.result = "result"; + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index aa77d29702..89931b8541 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,14 +18,10 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - CalculatedField save(CalculatedField calculatedField, SecurityUser user) throws ThingsboardException; CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 979c419003..2d3323e097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -86,7 +86,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -150,7 +150,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; @@ -179,7 +179,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ebda2cbf82..57d7afcaba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -136,7 +137,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 4bb67f9f0d..e90f52c5c0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -60,4 +60,6 @@ public interface CalculatedFieldService extends EntityDaoService { boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java new file mode 100644 index 0000000000..bcd22f9216 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +public class Argument { + + private EntityId entityId; + private String key; + private String type; + private String defaultValue; + + private int limit; + private long timeWindow; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java index 4575e414ac..be69b951d1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java @@ -105,14 +105,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return configNode; } - @Data - public static class Argument { - private EntityId entityId; - private String key; - private String type; - private String defaultValue; - } - @Data public static class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java index f733c35310..a3598cb59d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java @@ -39,7 +39,7 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldType getType(); - Map getArguments(); + Map getArguments(); BaseCalculatedFieldConfiguration.Output getOutput(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index fd669f7865..1ee881ae72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -195,6 +195,11 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldDao.existsByEntityId(tenantId, entityId); + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findById(tenantId, new CalculatedFieldId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index d1c9fd86bc..a6b7c2dea1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -34,4 +34,6 @@ public interface CalculatedFieldDao extends Dao { List removeAllByEntityId(TenantId tenantId, EntityId entityId); + boolean existsByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index bdc701070d..737a089a15 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -66,6 +66,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 43b51ce2ac..2082210899 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -880,7 +881,7 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index d5d025a46d..751d883896 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -149,7 +150,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { private CalculatedFieldConfiguration getCalculatedFieldConfig(EntityId referencedEntityId) { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 1c5e0d8f49..73d398bb39 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,6 +31,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -375,7 +376,7 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 8f1f16e631..1bd876eae0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.cf.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; @@ -1218,7 +1219,7 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); - SimpleCalculatedFieldConfiguration.Argument argument = new SimpleCalculatedFieldConfiguration.Argument(); + Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType("TIME_SERIES"); argument.setKey("temperature"); From 40299cab1ac2a54a73c09f342b48c5923d52d99b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 20 Nov 2024 17:34:07 +0200 Subject: [PATCH 034/438] added nofitication on update in telemetry service --- .../cf/CalculatedFieldExecutionService.java | 8 +++ .../cf/CalculatedFieldResult.java | 23 ++---- ...efaultCalculatedFieldExecutionService.java | 70 +++++++++++++++---- .../{entitiy => }/cf/RocksDBService.java | 4 +- .../cf => cf/ctx}/CalculatedFieldCtx.java | 5 +- .../cf => cf/ctx}/CalculatedFieldCtxId.java | 2 +- .../ctx/state}/CalculatedFieldState.java | 14 ++-- .../state}/ScriptCalculatedFieldState.java | 19 +++-- .../state}/SimpleCalculatedFieldState.java | 38 +++++++--- .../cf/DefaultTbCalculatedFieldService.java | 2 +- .../DefaultTelemetrySubscriptionService.java | 60 +++++++++++++++- .../CalculatedFieldControllerTest.java | 9 +-- .../server/dao/cf/CalculatedFieldService.java | 4 ++ .../common/data/cf/CalculatedField.java | 2 + .../cf/CalculatedFieldLinkConfiguration.java | 8 +-- .../data/cf/{ => configuration}/Argument.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 43 ++++++------ .../CalculatedFieldConfiguration.java | 9 ++- .../common/data/cf/configuration/Output.java | 29 ++++++++ .../ScriptCalculatedFieldConfiguration.java | 40 +++++++++++ .../SimpleCalculatedFieldConfiguration.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 16 ++++- .../server/dao/cf/CalculatedFieldLinkDao.java | 3 + .../dao/model/sql/CalculatedFieldEntity.java | 7 +- .../sql/cf/CalculatedFieldLinkRepository.java | 2 + ...efaultNativeCalculatedFieldRepository.java | 4 +- .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 ++ .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 9 +-- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 31 files changed, 349 insertions(+), 115 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/CalculatedFieldResult.java (53%) rename application/src/main/java/org/thingsboard/server/service/{entitiy => }/cf/RocksDBService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtx.java (84%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx}/CalculatedFieldCtxId.java (93%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/CalculatedFieldState.java (70%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/ScriptCalculatedFieldState.java (73%) rename application/src/main/java/org/thingsboard/server/service/{entitiy/cf => cf/ctx/state}/SimpleCalculatedFieldState.java (57%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/Argument.java (85%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/BaseCalculatedFieldConfiguration.java (81%) rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/CalculatedFieldConfiguration.java (81%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java rename common/data/src/main/java/org/thingsboard/server/common/data/cf/{ => configuration}/SimpleCalculatedFieldConfiguration.java (90%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6daa88d771..9a8b26d074 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,11 +15,19 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; +import java.util.Map; + public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + +// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index adb8b70e7e..4982445735 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -13,34 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import java.util.Map; + @Data public class CalculatedFieldResult { - private String name; private String type; private AttributeScope scope; - private String value; - - public static CalculatedFieldResult createAttributesResult(String name, AttributeScope scope, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "ATTRIBUTES"; - result.scope = scope; - result.value = value; - return result; - } + private Map resultMap; - public static CalculatedFieldResult createTimeSeriesResult(String name, String value) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.name = name; - result.type = "TIME_SERIES"; - result.value = value; - return result; + public CalculatedFieldResult() { } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0eeec02bb..250496830c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -29,13 +30,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.Argument; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -44,7 +44,10 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -54,12 +57,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtx; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldCtxId; -import org.thingsboard.server.service.entitiy.cf.CalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.RocksDBService; -import org.thingsboard.server.service.entitiy.cf.ScriptCalculatedFieldState; -import org.thingsboard.server.service.entitiy.cf.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; @@ -71,6 +73,9 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.DataConstants.SCOPE; @TbCoreComponent @Service @@ -84,6 +89,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; + private final TbClusterService clusterService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -194,6 +200,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + try { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + } catch (Exception e) { + log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); boolean shouldReinit = true; @@ -250,11 +267,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); + AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); argumentValues.put(key, value); + if (remaining.decrementAndGet() == 0) { + updateOrInitializeState(calculatedField, entityId, argumentValues); + } } @Override @@ -263,15 +284,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas callback.onFailure(t); } }, calculatedFieldCallbackExecutor)); - - updateOrInitializeState(calculatedField, entityId, argumentValues); - } private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), AttributeScope.SERVER_SCOPE, argument.getKey()), + attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), result -> result.map(entry -> (KvEntry) entry), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( @@ -296,7 +314,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - state.performCalculation(calculatedField.getConfiguration()); + CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + try { + String type = calculatedFieldResult.getType(); + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + } catch (Exception e) { + log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + } + } + + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { + ObjectNode jsonNodes = JacksonUtil.newObjectNode(); + calculatedFieldResult.getResultMap().forEach(jsonNodes::put); + return jsonNodes; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java rename to application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 2cf7aec18d..d6b2980042 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; @@ -94,4 +94,4 @@ public class RocksDBService { return map; } -} \ No newline at end of file +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java index 8a5e4cdf65..4b2a6c918f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import lombok.Data; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data public class CalculatedFieldCtx { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java index 3dc0dead36..a316c54b76 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 9997df947f..c25a6960ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -13,14 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.thingsboard.server.common.data.cf.BaseCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -30,15 +31,16 @@ import java.util.Map; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") }) public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map arguments, CalculatedFieldConfiguration calculatedFieldConfiguration) { - return arguments.keySet().containsAll(calculatedFieldConfiguration.getArguments().keySet()); + default boolean isValid(Map argumentValues, Map arguments) { + return argumentValues.keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 52435643cc..238e8005f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -13,23 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @Data +@Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private TbelInvokeService tbelInvokeService; - private Map arguments = new HashMap<>(); + public ScriptCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -37,11 +40,15 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - + if (arguments == null) { + this.arguments = new HashMap<>(); + } + this.arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + // TODO: implement return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index c004f9dd65..e984e300c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.entitiy.cf; +package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; import java.util.Map; @@ -28,8 +31,7 @@ import java.util.Map; public class SimpleCalculatedFieldState implements CalculatedFieldState { // TODO: use value object(TsKv) instead of string - private Map arguments = new HashMap<>(); - private String outputResult; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -38,29 +40,43 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { @Override public void initState(Map argumentValues) { - this.arguments = argumentValues; + if (arguments == null) { + arguments = new HashMap<>(); + } + arguments.putAll(argumentValues); } @Override public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - if (isValid(arguments, calculatedFieldConfiguration)) { - String expression = calculatedFieldConfiguration.getOutput().getExpression(); + Output output = calculatedFieldConfiguration.getOutput(); + Map arguments = calculatedFieldConfiguration.getArguments(); + + if (isValid(this.arguments, arguments)) { + CalculatedFieldResult result = new CalculatedFieldResult(); + String expression = output.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) .implicitMultiplication(true) - .variables(arguments.keySet()) + .variables(this.arguments.keySet()) .build(); customExpression.set(expr); } Map variables = new HashMap<>(); - arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); expr.setVariables(variables); - double result = expr.evaluate(); - this.outputResult = Double.toString(result); + + String expressionResult = String.valueOf(expr.evaluate()); + + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(Map.of(output.getName(), expressionResult)); + return result; } + return null; + // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 0cc644606f..4d28ff55ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 5513c41929..ac769bee9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,6 +32,8 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +42,7 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -47,9 +50,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,6 +82,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldExecutionService calculatedFieldExecutionService; private ExecutorService tsCallBackExecutor; @@ -87,12 +94,16 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TimeseriesService tsService, @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, - TbApiUsageStateService apiUsageStateService) { + TbApiUsageStateService apiUsageStateService, + CalculatedFieldService calculatedFieldService, + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; + this.calculatedFieldService = calculatedFieldService; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -179,6 +190,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -187,6 +199,49 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } + private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { + if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { + List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (!cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + List filteredTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .toList(); + + + Map updatedTelemetry = new HashMap<>(); + for (KvEntry telemetryEntry : filteredTelemetry) { + String key = telemetryEntry.getKey(); + if (telemetryEntry instanceof AttributeKvEntry) { + for (Map.Entry attribute : attributes.entrySet()) { + if (telemetryEntry.getKey().equals(attribute.getValue())) { + key = attribute.getKey(); + break; + } + } + } + if (telemetryEntry instanceof TsKvEntry) { + for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { + if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { + key = timeSeriesEntry.getKey(); + break; + } + } + } + updatedTelemetry.put(key, telemetryEntry.getValueAsString()); + } + + if (!updatedTelemetry.isEmpty()) { + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + } + }); + } + } + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -263,6 +318,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -270,6 +326,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); + updateTelemetryInCalculatedFields(tenantId, entityId, attributes); } @Override @@ -283,6 +340,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); + updateTelemetryInCalculatedFields(tenantId, entityId, ts); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 57d7afcaba..314dc2bdba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,11 +21,12 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.security.Authority; @@ -144,7 +145,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e90f52c5c0..e44ff0ba22 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -54,12 +54,16 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksById(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); + boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId); + boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index ceb1222fe2..e626c9d3d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index 02d668a67a..c5f81cd572 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -17,13 +17,13 @@ package org.thingsboard.server.common.data.cf; import lombok.Data; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private List attributes = new ArrayList<>(); - private List timeSeries = new ArrayList<>(); + private Map attributes = new HashMap<>(); + private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java similarity index 85% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index bcd22f9216..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.EntityId; @Data @@ -24,6 +25,7 @@ public class Argument { private EntityId entityId; private String key; private String type; + private AttributeScope scope; private String defaultValue; private int limit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index be69b951d1..7692d792f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -60,18 +62,20 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel @Override public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - arguments.values().stream() - .filter(argument -> argument.getEntityId().equals(entityId)) - .forEach(argument -> { - switch (argument.getType()) { - case "ATTRIBUTES": - linkConfiguration.getAttributes().add(argument.getKey()); - break; - case "TIME_SERIES": - linkConfiguration.getTimeSeries().add(argument.getKey()); - break; - } - }); + + for (Map.Entry entry : arguments.entrySet()) { + Argument argument = entry.getValue(); + if (argument.getEntityId().equals(entityId)) { + switch (argument.getType()) { + case "ATTRIBUTES": + linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); + break; + case "TIME_SERIES": + linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); + break; + } + } + } return linkConfiguration; } @@ -93,25 +97,21 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argumentNode.put("key", argument.getKey()); argumentNode.put("type", argument.getType()); + argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); }); if (output != null) { ObjectNode outputNode = configNode.putObject("output"); + outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); + outputNode.put("scope", String.valueOf(output.getScope())); outputNode.put("expression", output.getExpression()); } return configNode; } - @Data - public static class Output { - private String name; - private String type; - private String expression; - } - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { if (config == null || !config.isObject()) { return null; @@ -133,6 +133,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); + argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); @@ -142,7 +143,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); + output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); + output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); output.setExpression(outputNode.get("expression").asText()); this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java similarity index 81% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index a3598cb59d..155015028f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -13,13 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; @@ -32,7 +34,8 @@ import java.util.UUID; property = "type" ) @JsonSubTypes({ - @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE") + @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { @@ -41,7 +44,7 @@ public interface CalculatedFieldConfiguration { Map getArguments(); - BaseCalculatedFieldConfiguration.Output getOutput(); + Output getOutput(); @JsonIgnore List getReferencedEntities(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java new file mode 100644 index 0000000000..683e372ebc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +public class Output { + + private String name; + private String type; + private AttributeScope scope; + private String expression; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..a24328b4c9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public ScriptCalculatedFieldConfiguration() { + super(); + } + + public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.SCRIPT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index 327f9cdc75..af11d2f5d8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.cf; +package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 1ee881ae72..cf5f79c5fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -21,7 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -173,6 +173,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId); } + @Override + public List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findAllCalculatedFieldLinksByEntityId, entityId [{}]", entityId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); + } + @Override public ListenableFuture> findAllCalculatedFieldLinksByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.trace("Executing findAllCalculatedFieldLinksByIdAsync, calculatedFieldId [{}]", calculatedFieldId); @@ -195,6 +201,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); } + @Override + public boolean referencedInAnyCalculatedFieldIncludingEntityId(TenantId tenantId, EntityId referencedEntityId) { + return calculatedFieldDao.findAllByTenantId(tenantId).stream() + .map(CalculatedField::getConfiguration) + .map(CalculatedFieldConfiguration::getReferencedEntities) + .anyMatch(referencedEntities -> referencedEntities.contains(referencedEntityId)); + } + @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 34f2129bd7..549db510ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.cf; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -29,6 +30,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByCalculatedFieldId(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); List findAll(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 3c45a81cb5..6500d2a1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,9 +24,10 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -122,6 +123,8 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem switch (CalculatedFieldType.valueOf(type)) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index 61c4026cca..d7325df8d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -25,4 +25,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantIdAndCalculatedFieldId(UUID tenantId, UUID calculatedFieldId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 417a468b2c..eebca14b6e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,11 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 417b529dc9..29492a10cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); + } + @Override public ListenableFuture> findCalculatedFieldLinksByCalculatedFieldIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return service.submit(() -> findCalculatedFieldLinksByCalculatedFieldId(tenantId, calculatedFieldId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2082210899..2012090264 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,10 +30,11 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -888,7 +889,7 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 751d883896..2bdb1b8897 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,11 +23,12 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.cf.CalculatedFieldService; @@ -157,7 +158,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 73d398bb39..b58f739462 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,10 +31,11 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -383,7 +384,7 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 1bd876eae0..38bd21170a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,10 +39,11 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.Argument; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -1226,7 +1227,7 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); - SimpleCalculatedFieldConfiguration.Output output = new SimpleCalculatedFieldConfiguration.Output(); + Output output = new Output(); output.setType("TIME_SERIES"); output.setExpression("T - (100 - H) / 5"); From c75603f57c0d093d14cb7c3cf4fe187082f70ca5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 15:45:39 +0200 Subject: [PATCH 035/438] added implementation for script type --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 43 ++++++--- .../state/CalculatedFieldScriptEngine.java | 29 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 12 ++- .../CalculatedFieldTbelScriptEngine.java | 90 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 59 ++++++++++-- .../ctx/state/SimpleCalculatedFieldState.java | 22 ++--- .../DefaultTelemetrySubscriptionService.java | 47 +++++----- .../thingsboard/script/api/ScriptType.java | 2 +- 10 files changed, 248 insertions(+), 63 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 9a8b26d074..d4e0d1da84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; @@ -26,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - -// void onEntityProfileUpdate(TransportProtos.CalculatedFieldEntityProfileUpdateMsgProto proto, TbCallback callback); + void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 4982445735..87f1d08a84 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -25,7 +25,7 @@ public class CalculatedFieldResult { private String type; private AttributeScope scope; - private Map resultMap; + private Map resultMap; public CalculatedFieldResult() { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 250496830c..c83bd7a71a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -30,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -90,6 +92,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; private final TbClusterService clusterService; + private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -201,7 +204,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); @@ -266,13 +269,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - String value = result.map(KvEntry::getValueAsString).orElse(argument.getDefaultValue()); - argumentValues.put(key, value); + // todo: should be rewritten implementation for default value + argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -300,7 +303,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -314,10 +317,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculatedFieldResult result = state.performCalculation(calculatedField.getConfiguration()); - if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); - } + ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + } + }, MoreExecutors.directExecutor()); + } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { @@ -325,8 +339,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; - ObjectNode jsonNodes = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(jsonNodes)); + ObjectNode payload = createJsonPayload(calculatedFieldResult); + TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); @@ -334,9 +348,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { - ObjectNode jsonNodes = JacksonUtil.newObjectNode(); - calculatedFieldResult.getResultMap().forEach(jsonNodes::put); - return jsonNodes; + ObjectNode payload = JacksonUtil.newObjectNode(); + Map resultMap = calculatedFieldResult.getResultMap(); + resultMap.forEach((k, v) -> payload.set(k, JacksonUtil.convertValue(v, JsonNode.class))); + return payload; } private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java new file mode 100644 index 0000000000..6b54536019 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +public interface CalculatedFieldScriptEngine { + + ListenableFuture executeScriptAsync(Map arguments); + + void destroy(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index c25a6960ac..dffcf09820 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -18,9 +18,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -39,12 +43,12 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { + default boolean isValid(Map argumentValues, Map arguments) { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration); + ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java new file mode 100644 index 0000000000..7e8376be8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.ScriptType; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import javax.script.ScriptException; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +@Slf4j +public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEngine { + + private final TbelInvokeService tbelInvokeService; + + private final UUID scriptId; + private final TenantId tenantId; + + public CalculatedFieldTbelScriptEngine(TenantId tenantId, TbelInvokeService tbelInvokeService, String script, String... argNames) { + this.tenantId = tenantId; + this.tbelInvokeService = tbelInvokeService; + try { + this.scriptId = this.tbelInvokeService.eval(tenantId, ScriptType.CALCULATED_FIELD_SCRIPT, script, argNames).get(); + } catch (Exception e) { + Throwable t = e; + if (e instanceof ExecutionException) { + t = e.getCause(); + } + throw new IllegalArgumentException("Can't compile script: " + t.getMessage(), t); + } + } + + @Override + public ListenableFuture executeScriptAsync(Map arguments) { + log.trace("execute script async, arguments {}", arguments); + Object[] args = new Object[arguments.size()]; + int index = 0; + for (KvEntry entry : arguments.values()) { + switch (entry.getDataType()) { + case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); + case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); + case LONG -> args[index] = entry.getLongValue().orElse(null); + case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); + default -> args[index] = entry.getValueAsString(); + } + index++; + } + return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), + o -> { + try { + return Futures.immediateFuture(o); + } catch (Exception e) { + if (e.getCause() instanceof ScriptException) { + return Futures.immediateFailedFuture(e.getCause()); + } else if (e.getCause() instanceof RuntimeException) { + return Futures.immediateFailedFuture(new ScriptException(e.getCause().getMessage())); + } else { + return Futures.immediateFailedFuture(new ScriptException(e)); + } + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void destroy() { + tbelInvokeService.release(this.scriptId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 238e8005f2..363c237df5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,10 +15,19 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -28,7 +37,10 @@ import java.util.Map; @Slf4j public class ScriptCalculatedFieldState implements CalculatedFieldState { - private Map arguments = new HashMap<>(); + @JsonIgnore + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + private Map arguments = new HashMap<>(); public ScriptCalculatedFieldState() { } @@ -39,7 +51,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { this.arguments = new HashMap<>(); } @@ -47,9 +59,46 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { - // TODO: implement - return null; + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + if (calculatedFieldScriptEngine == null) { + initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + } + + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); + + return Futures.transform(resultFuture, result -> { + Output output = calculatedFieldConfiguration.getOutput(); + Map resultMap = new HashMap<>(); + + if (result instanceof Map) { + Map map = JacksonUtil.convertValue(result, Map.class); + if (map != null) { + resultMap.putAll(map); + } + } else { + resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); + } + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return calculatedFieldResult; + }, MoreExecutors.directExecutor()); + } + + private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + calculatedFieldConfiguration.getOutput().getExpression(), + arguments.keySet().toArray(new String[0]) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e984e300c6..725da6c7a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -30,8 +35,7 @@ import java.util.Map; @Data public class SimpleCalculatedFieldState implements CalculatedFieldState { - // TODO: use value object(TsKv) instead of string - private Map arguments; + private Map arguments; @Override public CalculatedFieldType getType() { @@ -39,7 +43,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } @@ -47,7 +51,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { } @Override - public CalculatedFieldResult performCalculation(CalculatedFieldConfiguration calculatedFieldConfiguration) { + public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); @@ -64,19 +68,17 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); expr.setVariables(variables); - String expressionResult = String.valueOf(expr.evaluate()); + double expressionResult = expr.evaluate(); result.setType(output.getType()); result.setScope(output.getScope()); result.setResultMap(Map.of(output.getName(), expressionResult)); - return result; + return Futures.immediateFuture(result); } - return null; - // TODO: handle what happens when not valid } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ac769bee9e..76c1383f6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -69,6 +69,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -207,32 +208,28 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); Map timeSeries = link.getConfiguration().getTimeSeries(); - List filteredTelemetry = telemetry.stream() + Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .toList(); - - - Map updatedTelemetry = new HashMap<>(); - for (KvEntry telemetryEntry : filteredTelemetry) { - String key = telemetryEntry.getKey(); - if (telemetryEntry instanceof AttributeKvEntry) { - for (Map.Entry attribute : attributes.entrySet()) { - if (telemetryEntry.getKey().equals(attribute.getValue())) { - key = attribute.getKey(); - break; - } - } - } - if (telemetryEntry instanceof TsKvEntry) { - for (Map.Entry timeSeriesEntry : timeSeries.entrySet()) { - if (telemetryEntry.getKey().equals(timeSeriesEntry.getValue())) { - key = timeSeriesEntry.getKey(); - break; - } - } - } - updatedTelemetry.put(key, telemetryEntry.getValueAsString()); - } + .collect(Collectors.toMap( + entry -> { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + }, + entry -> entry, + (v1, v2) -> v1 + )); if (!updatedTelemetry.isEmpty()) { calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java index cdcdf815d0..7f8c513957 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/ScriptType.java @@ -16,5 +16,5 @@ package org.thingsboard.script.api; public enum ScriptType { - RULE_NODE_SCRIPT + RULE_NODE_SCRIPT, CALCULATED_FIELD_SCRIPT } From 6ed4abb9038eb424c129965d99fae7ed828c633a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 21 Nov 2024 17:39:16 +0200 Subject: [PATCH 036/438] separated expression from output --- ...efaultCalculatedFieldExecutionService.java | 4 +-- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../CalculatedFieldControllerTest.java | 8 +++--- .../BaseCalculatedFieldConfiguration.java | 27 +++++++++++++++---- .../CalculatedFieldConfiguration.java | 2 ++ .../common/data/cf/configuration/Output.java | 1 - ...efaultNativeCalculatedFieldRepository.java | 5 +++- .../server/dao/service/AssetServiceTest.java | 6 +++-- .../service/CalculatedFieldServiceTest.java | 8 +++--- .../dao/service/CustomerServiceTest.java | 6 +++-- .../server/dao/service/DeviceServiceTest.java | 6 +++-- 12 files changed, 54 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c83bd7a71a..6aa9997d1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -253,9 +253,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean outputExpressionChanged = !oldConfig.getOutput().getExpression().equals(newConfig.getOutput().getExpression()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || outputExpressionChanged; + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void fetchCalculatedFields() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 363c237df5..cbee8dc902 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -96,7 +96,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getOutput().getExpression(), + calculatedFieldConfiguration.getExpression(), arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 725da6c7a7..08c4741de5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { if (isValid(this.arguments, arguments)) { CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = output.getExpression(); + String expression = calculatedFieldConfiguration.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 314dc2bdba..77ca268d12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -21,10 +21,10 @@ import org.junit.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; @@ -145,9 +145,11 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 7692d792f8..55a43a00ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -40,6 +40,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel private final ObjectMapper mapper = new ObjectMapper(); protected Map arguments; + protected String expression; protected Output output; public BaseCalculatedFieldConfiguration() { @@ -48,6 +49,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); this.arguments = calculatedFieldConfig.getArguments(); + this.expression = calculatedFieldConfig.getExpression(); this.output = calculatedFieldConfig.getOutput(); } @@ -101,12 +103,17 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("defaultValue", argument.getDefaultValue()); }); + if (expression != null) { + configNode.put("expression", expression); + } + if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); outputNode.put("type", output.getType()); - outputNode.put("scope", String.valueOf(output.getScope())); - outputNode.put("expression", output.getExpression()); + if (output.getScope() != null) { + outputNode.put("scope", String.valueOf(output.getScope())); + } } return configNode; @@ -133,20 +140,30 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel } argument.setKey(argumentNode.get("key").asText()); argument.setType(argumentNode.get("type").asText()); - argument.setScope(AttributeScope.valueOf(argumentNode.get("scope").asText())); + JsonNode scope = argumentNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + argument.setScope(AttributeScope.valueOf(scope.asText())); + } argument.setDefaultValue(argumentNode.get("defaultValue").asText()); arguments.put(key, argument); }); } this.setArguments(arguments); + JsonNode expressionNode = config.get("expression"); + if (expressionNode != null && expressionNode.isTextual()) { + this.setExpression(expressionNode.asText()); + } + JsonNode outputNode = config.get("output"); if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); output.setType(outputNode.get("type").asText()); - output.setScope(AttributeScope.valueOf(outputNode.get("scope").asText())); - output.setExpression(outputNode.get("expression").asText()); + JsonNode scope = outputNode.get("scope"); + if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { + output.setScope(AttributeScope.valueOf(scope.asText())); + } this.setOutput(output); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 155015028f..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -44,6 +44,8 @@ public interface CalculatedFieldConfiguration { Map getArguments(); + String getExpression(); + Output getOutput(); @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 683e372ebc..46257d1ccc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -24,6 +24,5 @@ public class Output { private String name; private String type; private AttributeScope scope; - private String expression; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index eebca14b6e..fc40f72c93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -25,10 +25,11 @@ import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; @@ -138,6 +139,8 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF switch (type) { case SIMPLE: return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT: + return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); default: throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 2012090264..87f2cb1f45 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -30,9 +30,9 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -889,9 +889,11 @@ public class AssetServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 2bdb1b8897..77ed026b1d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -23,10 +23,10 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -158,9 +158,11 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index b58f739462..94c8440057 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -31,9 +31,9 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; @@ -384,9 +384,11 @@ public class CustomerServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 38bd21170a..d5394a3494 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -39,9 +39,9 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; @@ -1227,9 +1227,11 @@ public class DeviceServiceTest extends AbstractServiceTest { config.setArguments(Map.of("T", argument)); + config.setExpression("T - (100 - H) / 5"); + Output output = new Output(); + output.setName("output"); output.setType("TIME_SERIES"); - output.setExpression("T - (100 - H) / 5"); config.setOutput(output); From 31103e90e3e3cb5f440823872f2a0c85cdac6b10 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 22 Nov 2024 17:17:59 +0200 Subject: [PATCH 037/438] added implementation to handle update entity profiles events --- .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 74 ++++++++++++++++--- .../CalculatedFieldTbelScriptEngine.java | 14 +--- .../ctx/state/ScriptCalculatedFieldState.java | 13 +--- .../entitiy/EntityStateSourcingListener.java | 31 ++++---- .../queue/DefaultTbClusterService.java | 44 ++++++++++- .../queue/DefaultTbCoreConsumerService.java | 19 ++++- .../server/cluster/TbClusterService.java | 3 + .../server/dao/cf/CalculatedFieldService.java | 2 + common/proto/src/main/proto/queue.proto | 34 ++++++--- .../server/dao/asset/BaseAssetService.java | 4 +- .../dao/cf/BaseCalculatedFieldService.java | 14 +++- .../server/dao/cf/CalculatedFieldDao.java | 3 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++ 15 files changed, 202 insertions(+), 64 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d4e0d1da84..6b7b12655f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,4 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6aa9997d1f..9edea50a05 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -27,12 +27,14 @@ import jakarta.annotation.PreDestroy; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -44,8 +46,14 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -210,7 +218,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update time series for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + } + } + + @Override + public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); + cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); + List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); + rocksDBService.deleteAll(ctxIdsToDelete); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) + .forEach(cfId -> { + CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + }); + + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) + .stream() + .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) + .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); } } @@ -271,10 +310,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { @Override public void onSuccess(Optional result) { - // todo: should be rewritten implementation for default value argumentValues.put(key, result.orElse(null)); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); @@ -289,20 +327,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument) { + private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; return switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, argument.getEntityId(), argument.getScope(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, argument.getEntityId(), argument.getKey()), - result -> result.map(entry -> (KvEntry) entry), + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of( + new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) + )), MoreExecutors.directExecutor()); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -322,7 +378,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), calculatedField.getEntityId(), result); + pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7e8376be8e..5cc58b2a95 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -55,18 +54,7 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng @Override public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = new Object[arguments.size()]; - int index = 0; - for (KvEntry entry : arguments.values()) { - switch (entry.getDataType()) { - case BOOLEAN -> args[index] = entry.getBooleanValue().orElse(null); - case DOUBLE -> args[index] = entry.getDoubleValue().orElse(null); - case LONG -> args[index] = entry.getLongValue().orElse(null); - case JSON -> args[index] = entry.getJsonValue().map(JacksonUtil::toJsonNode).orElse(null); - default -> args[index] = entry.getValueAsString(); - } - index++; - } + Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index cbee8dc902..5c984a8d16 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -72,16 +72,9 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return Futures.transform(resultFuture, result -> { Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = new HashMap<>(); - - if (result instanceof Map) { - Map map = JacksonUtil.convertValue(result, Map.class); - if (map != null) { - resultMap.putAll(map); - } - } else { - resultMap.put(output.getName(), JacksonUtil.convertValue(result, Object.class)); - } + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); calculatedFieldResult.setType(output.getType()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 1b541bcd5d..154f5f4833 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; @@ -51,8 +52,6 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @@ -67,8 +66,6 @@ public class EntityStateSourcingListener { private final TbClusterService tbClusterService; private final TenantService tenantService; - private final CalculatedFieldService calculatedFieldService; - private final DeviceProfileService deviceProfileService; @PostConstruct public void init() { @@ -88,7 +85,10 @@ public class EntityStateSourcingListener { ComponentLifecycleEvent lifecycleEvent = isCreated ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED; switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET -> { + onAssetUpdate(event.getEntity(), event.getOldEntity()); + } + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -106,7 +106,7 @@ public class EntityStateSourcingListener { onTenantProfileUpdate(tenantProfile, lifecycleEvent); } case DEVICE -> { - onDeviceUpdate(tenantId, event.getEntity(), event.getOldEntity()); + onDeviceUpdate(event.getEntity(), event.getOldEntity()); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -245,23 +245,24 @@ public class EntityStateSourcingListener { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } - private void onDeviceUpdate(TenantId tenantId, Object entity, Object oldEntity) { + private void onDeviceUpdate(Object entity, Object oldEntity) { Device device = (Device) entity; Device oldDevice = null; if (oldEntity instanceof Device) { oldDevice = (Device) oldEntity; - // TODO: move verification of device type to cluster service - if (!oldDevice.getType().equals(device.getType())) { - DeviceProfile profile = deviceProfileService.findDeviceProfileByName(tenantId, device.getType()); - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profile.getId()); - if (cfExistsByProfile) { - // TODO: send device type updated msg to core - } - } } tbClusterService.onDeviceUpdated(device, oldDevice); } + private void onAssetUpdate(Object entity, Object oldEntity) { + Asset asset = (Asset) entity; + Asset oldAsset = null; + if (oldEntity instanceof Asset) { + oldAsset = (Asset) oldEntity; + } + tbClusterService.onAssetUpdated(asset, oldAsset); + } + private void onEdgeEvent(TenantId tenantId, EntityId entityId, Object entity, ComponentLifecycleEvent lifecycleEvent) { if (entity instanceof Edge) { tbClusterService.onEdgeStateChangeEvent(new ComponentLifecycleMsg(tenantId, entityId, lifecycleEvent)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index f7c8230716..58a16d9c0a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,6 +68,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -149,6 +150,7 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; + private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -609,7 +611,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - if (deviceNameChanged || !device.getType().equals(old.getType())) { + boolean deviceTypeChanged = !device.getType().equals(old.getType()); + if (deviceTypeChanged) { + handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + } + if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } @@ -618,6 +624,26 @@ public class DefaultTbClusterService implements TbClusterService { otaPackageStateService.update(device, old); } + @Override + public void onAssetUpdated(Asset asset, Asset old) { + var created = old == null; + broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + if (old != null) { + boolean assetTypeChanged = !asset.getType().equals(old.getType()); + if (assetTypeChanged) { + handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + } + } + broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + if (cfExistsByProfile) { + sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -775,4 +801,20 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } + private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2d3323e097..3caaab6613 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -157,6 +158,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; + private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -202,6 +204,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -315,6 +318,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = deviceActivityEventsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -677,6 +682,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index f173005107..131b50a52c 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; @@ -97,6 +98,8 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); + void onAssetUpdated(Asset asset, Asset old); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index e44ff0ba22..1e64fdac60 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -36,6 +36,8 @@ public interface CalculatedFieldService extends EntityDaoService { ListenableFuture findCalculatedFieldByIdAsync(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 95e4fa8601..bf49a68c80 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -771,6 +771,29 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } +message CalculatedFieldMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + bool added = 5; + bool updated = 6; + bool deleted = 7; +} + +message EntityProfileUpdateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 oldProfileIdMSB = 7; + int64 oldProfileIdLSB = 8; + int64 newProfileIdMSB = 9; + int64 newProfileIdLSB = 10; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1267,16 +1290,6 @@ message ToDeviceActorNotificationMsgProto { DeviceDeleteMsgProto deviceDeleteMsg = 8; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - /** TB Core to Version Control Service */ @@ -1513,6 +1526,7 @@ message ToCoreMsg { DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 2135792174..7242cfde68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -177,8 +177,8 @@ public class BaseAssetService extends AbstractCachedEntityService findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldIdsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); @@ -133,7 +141,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements public int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteAllCalculatedFieldsByEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - validateId(entityId.getId(), id -> "Incorrect entityId " + id); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); List calculatedFields = calculatedFieldDao.removeAllByEntityId(tenantId, entityId); return calculatedFields.size(); } @@ -212,7 +220,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements @Override public boolean existsCalculatedFieldByEntityId(TenantId tenantId, EntityId entityId) { return calculatedFieldDao.existsByEntityId(tenantId, entityId); - }; + } @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index a6b7c2dea1..5b3bcc2750 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.cf; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -28,6 +29,8 @@ public interface CalculatedFieldDao extends Dao { List findAllByTenantId(TenantId tenantId); + List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 333057e8c5..9aa0aee428 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; import java.util.List; @@ -25,6 +26,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 737a089a15..e3762f6157 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -49,6 +50,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findCalculatedFieldIdsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 2c7c6f0c5edfe140a1ce0e528824ec58c48b5647 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 08:57:09 +0200 Subject: [PATCH 038/438] fixed tests for cluster service --- .../server/service/queue/DefaultTbClusterServiceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java index 25fe589a08..cb61980cf4 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbClusterServiceTest.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; @@ -102,6 +103,8 @@ public class DefaultTbClusterServiceTest { protected TbRuleEngineProducerService ruleEngineProducerService; @MockBean protected TbTransactionalCache edgeCache; + @MockBean + protected CalculatedFieldService calculatedFieldService; @SpyBean protected TopicService topicService; From c6d91c4ce870886e4af404246020dba15e089dd7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 25 Nov 2024 17:28:44 +0200 Subject: [PATCH 039/438] added last records type of cf --- ...efaultCalculatedFieldExecutionService.java | 84 ++++++++++++++--- .../service/cf/ctx/state/ArgumentEntry.java | 30 ++++++ .../cf/ctx/state/CalculatedFieldState.java | 10 +- .../cf/ctx/state/CalculationContext.java | 35 +++++++ .../LastRecordsCalculatedFieldState.java | 91 +++++++++++++++++++ .../ctx/state/ScriptCalculatedFieldState.java | 17 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 13 ++- .../common/data/cf/CalculatedFieldType.java | 2 +- .../data/cf/configuration/Argument.java | 1 + .../BaseCalculatedFieldConfiguration.java | 16 +++- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 ++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 16 ++-- ...efaultNativeCalculatedFieldRepository.java | 14 ++- 14 files changed, 323 insertions(+), 48 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9edea50a05..a985f093c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -48,12 +48,16 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -69,7 +73,10 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; @@ -84,6 +91,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -109,6 +117,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -215,7 +225,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), updatedTelemetry); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntry(entry.getValue()); + if (entry.getValue() instanceof TsKvEntry) { + argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); + } + return argumentEntry; + } + )); + updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -308,12 +330,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); - Map argumentValues = new HashMap<>(); + Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(tenantId, argument, entityId), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { @Override - public void onSuccess(Optional result) { - argumentValues.put(key, result.orElse(null)); + public void onSuccess(ArgumentEntry result) { + argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { updateOrInitializeState(calculatedField, entityId, argumentValues); } @@ -327,10 +349,37 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture> fetchArgumentValue(TenantId tenantId, Argument argument, EntityId targetEntityId) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + TenantId tenantId = calculatedField.getTenantId(); + EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - return switch (argument.getType()) { + EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + ? cfEntityId + : argumentEntityId; + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + return fetchLastRecords(tenantId, entityId, argument); + } + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + long startTs = Math.max(argument.getStartTs(), 0); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long endTs = startTs + timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(lastRecordsFuture, lastRecords -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + argumentEntry.setKvEntries(lastRecords); + return argumentEntry; + }, calculatedFieldExecutor); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + ListenableFuture> kvEntryFuture = switch (argument.getType()) { case "ATTRIBUTES" -> Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), result -> result.or(() -> Optional.of( @@ -342,9 +391,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas result -> result.or(() -> Optional.of( new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) )), - MoreExecutors.directExecutor()); + calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; + return Futures.transform(kvEntryFuture, kvEntry -> { + ArgumentEntry argumentEntry = new ArgumentEntry(); + if (kvEntry.isPresent()) { + argumentEntry.setKvEntry(kvEntry.orElse(null)); + } + return argumentEntry; + }, calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -359,7 +415,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); @@ -373,7 +429,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - ListenableFuture resultFuture = state.performCalculation(calculatedField.getTenantId(), calculatedField.getConfiguration(), tbelInvokeService); + CalculationContext ctx = CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .configuration(calculatedField.getConfiguration()) + .tbelInvokeService(tbelInvokeService) + .build(); + ListenableFuture resultFuture = state.performCalculation(ctx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -414,6 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); + case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java new file mode 100644 index 0000000000..29e3417bde --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class ArgumentEntry { + + private KvEntry kvEntry; + private List kvEntries; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index dffcf09820..e4c4440921 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -19,11 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -36,7 +33,8 @@ import java.util.Map; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { @@ -47,8 +45,8 @@ public interface CalculatedFieldState { return argumentValues.keySet().containsAll(arguments.keySet()); } - void initState(Map argumentValues); + void initState(Map argumentValues); - ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService); + ListenableFuture performCalculation(CalculationContext ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java new file mode 100644 index 0000000000..656763ea48 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.Map; + +@Data +@Builder +public class CalculationContext { + + private TenantId tenantId; + private CalculatedFieldConfiguration configuration; + private TbelInvokeService tbelInvokeService; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java new file mode 100644 index 0000000000..2f26d71f91 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class LastRecordsCalculatedFieldState implements CalculatedFieldState { + + private Map> arguments; + + public LastRecordsCalculatedFieldState() { + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } + + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } + argumentValues.forEach((key, argumentEntry) -> { + List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); + tsKvEntryList.addAll(argumentEntry.getKvEntries()); + }); + } + + + @Override + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration configuration = ctx.getConfiguration(); + Map configArguments = configuration.getArguments(); + Output output = configuration.getOutput(); + + Map resultMap = new HashMap<>(); + + arguments.replaceAll((key, entries) -> { + int limit = configArguments.get(key).getLimit(); + List limitedEntries = entries.stream() + .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) + .limit(limit) + .collect(Collectors.toList()); + + Map valueWithTs = limitedEntries.stream() + .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); + resultMap.put(key, valueWithTs); + + return limitedEntries; + }); + + CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); + calculatedFieldResult.setType(output.getType()); + calculatedFieldResult.setScope(output.getScope()); + calculatedFieldResult.setResultMap(resultMap); + + return Futures.immediateFuture(calculatedFieldResult); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 5c984a8d16..0e9b00ad7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -40,7 +40,7 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments = new HashMap<>(); + private Map arguments; public ScriptCalculatedFieldState() { } @@ -50,22 +50,27 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { - this.arguments = new HashMap<>(); + arguments = new HashMap<>(); } - this.arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } + @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + if (tbelInvokeService == null) { throw new IllegalArgumentException("TBEL script engine is disabled!"); } if (calculatedFieldScriptEngine == null) { - initEngine(tenantId, calculatedFieldConfiguration, tbelInvokeService); + initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); } ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 08c4741de5..d3d3b5f636 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -20,12 +20,10 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,21 +35,26 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { private Map arguments; + public SimpleCalculatedFieldState() { + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } @Override - public void initState(Map argumentValues) { + public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll(argumentValues); + argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); } @Override - public ListenableFuture performCalculation(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { + public ListenableFuture performCalculation(CalculationContext ctx) { + CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); + Output output = calculatedFieldConfiguration.getOutput(); Map arguments = calculatedFieldConfiguration.getArguments(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 89173b35b9..63b6d8d1dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT + SIMPLE, SCRIPT, LAST_RECORDS } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..0d70591a38 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,6 +29,7 @@ public class Argument { private String defaultValue; private int limit; + private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 55a43a00ce..f7cc53b7cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -101,6 +101,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("type", argument.getType()); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); + argumentNode.put("limit", String.valueOf(argument.getLimit())); + argumentNode.put("startTs", String.valueOf(argument.getStartTs())); + argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); if (expression != null) { @@ -144,7 +147,18 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); } - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + if (argumentNode.hasNonNull("defaultValue")) { + argument.setDefaultValue(argumentNode.get("defaultValue").asText()); + } + if (argumentNode.hasNonNull("limit")) { + argument.setLimit(argumentNode.get("limit").asInt()); + } + if (argumentNode.hasNonNull("startTs")) { + argument.setStartTs(argumentNode.get("startTs").asLong()); + } + if (argumentNode.hasNonNull("timeWindow")) { + argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); + } arguments.put(key, argument); }); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..15f7a82c40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,7 +35,8 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), + @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java new file mode 100644 index 0000000000..c3f5804227 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; + +import java.util.UUID; + +@Data +public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { + + public LastRecordsCalculatedFieldConfiguration() { + } + + public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { + super(config, entityType, entityId); + } + + @Override + public CalculatedFieldType getType() { + return CalculatedFieldType.LAST_RECORDS; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6500d2a1e7..b06676f70b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -24,8 +24,9 @@ import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -120,14 +121,11 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (CalculatedFieldType.valueOf(type)) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fc40f72c93..2acd4d75c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -136,14 +137,11 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF } private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - switch (type) { - case SIMPLE: - return new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT: - return new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - default: - throw new IllegalArgumentException("Unsupported calculated field type: " + type + "!"); - } + return switch (type) { + case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); + case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); + case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); + }; } } From 80d9b22068ea44a5fc3d4bada2baa5c704d24246 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 11:31:49 +0200 Subject: [PATCH 040/438] added argument entry interface --- ...efaultCalculatedFieldExecutionService.java | 66 +++++++------------ .../service/cf/ctx/state/ArgumentEntry.java | 19 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 48 ++++++++++++++ .../cf/ctx/state/CalculationContext.java | 6 +- .../service/cf/ctx/state/KvArgumentEntry.java | 31 +++++++++ .../ctx/state/LastRecordsArgumentEntry.java | 33 ++++++++++ .../LastRecordsCalculatedFieldState.java | 21 ++---- .../ctx/state/ScriptCalculatedFieldState.java | 60 ++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 31 ++------- 9 files changed, 185 insertions(+), 130 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index a985f093c9..322443c848 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -191,19 +191,19 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(tenantId, cf, entityId, callback); + initializeStateForEntity(cf, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(tenantId, cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(tenantId, cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -226,17 +226,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntry(entry.getValue()); - if (entry.getValue() instanceof TsKvEntry) { - argumentEntry.setKvEntries(List.of((TsKvEntry) entry.getValue())); - } - return argumentEntry; - } - )); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { @@ -254,11 +244,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - List cfIdsOfOldProfile = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId); - cfIdsOfOldProfile.forEach(id -> states.remove(new CalculatedFieldCtxId(id.getId(), entityId.getId()))); - List ctxIdsToDelete = cfIdsOfOldProfile.stream().map(cfId -> JacksonUtil.writeValueAsString(new CalculatedFieldCtxId(cfId.getId(), entityId.getId()))).toList(); - rocksDBService.deleteAll(ctxIdsToDelete); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); @@ -269,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(tenantId, cf, entityId, callback)); + .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -328,11 +313,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(TenantId tenantId, CalculatedField calculatedField, EntityId entityId, TbCallback callback) { + private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { Map arguments = calculatedField.getConfiguration().getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); @@ -349,12 +334,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, Argument argument) { + private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedField.getTenantId(); - EntityId cfEntityId = calculatedField.getEntityId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) - ? cfEntityId + ? targetEntityId : argumentEntityId; if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { return fetchLastRecords(tenantId, entityId, argument); @@ -371,11 +355,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, lastRecords -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - argumentEntry.setKvEntries(lastRecords); - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -394,13 +374,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - ArgumentEntry argumentEntry = new ArgumentEntry(); - if (kvEntry.isPresent()) { - argumentEntry.setKvEntry(kvEntry.orElse(null)); - } - return argumentEntry; - }, calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -429,12 +403,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.put(ctxId, calculatedFieldCtx); rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); - CalculationContext ctx = CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .configuration(calculatedField.getConfiguration()) - .tbelInvokeService(tbelInvokeService) - .build(); - ListenableFuture resultFuture = state.performCalculation(ctx); + ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { @@ -464,6 +433,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculationContext buildCalculationContext(CalculatedField calculatedField) { + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + return CalculationContext.builder() + .tenantId(calculatedField.getTenantId()) + .arguments(configuration.getArguments()) + .output(configuration.getOutput()) + .expression(configuration.getExpression()) + .tbelInvokeService(tbelInvokeService) + .build(); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 29e3417bde..3097056d11 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,16 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -@Data -public class ArgumentEntry { +public interface ArgumentEntry { - private KvEntry kvEntry; - private List kvEntries; + Object getValue(); + + static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { + if (kvEntry instanceof TsKvEntry tsKvEntry) { + return new LastRecordsArgumentEntry(List.of(tsKvEntry)); + } else { + return new KvArgumentEntry(kvEntry); + } + } + + static ArgumentEntry createArgumentEntry(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java new file mode 100644 index 0000000000..bac318a1b9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; + +public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + + protected Map arguments; + + public BaseCalculatedFieldState() { + } + + @Override + public void initState(Map argumentValues) { + if (arguments == null) { + arguments = new HashMap<>(); + } +// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); + } + + protected CalculatedFieldResult buildResult(Output output, Map resultMap) { + CalculatedFieldResult result = new CalculatedFieldResult(); + result.setType(output.getType()); + result.setScope(output.getScope()); + result.setResultMap(resultMap); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java index 656763ea48..aaabaec0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Builder; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -29,7 +31,9 @@ import java.util.Map; public class CalculationContext { private TenantId tenantId; - private CalculatedFieldConfiguration configuration; + private Map arguments; + private Output output; + private String expression; private TbelInvokeService tbelInvokeService; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java new file mode 100644 index 0000000000..0bd21d452e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +public class KvArgumentEntry implements ArgumentEntry { + + private final KvEntry kvEntry; + + @Override + public Object getValue() { + return kvEntry; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java new file mode 100644 index 0000000000..8729f022fa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +public class LastRecordsArgumentEntry implements ArgumentEntry { + + private final List kvEntries; + + @Override + public Object getValue() { + return kvEntries; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 2f26d71f91..0428d0823d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -33,7 +35,7 @@ import java.util.Map; import java.util.stream.Collectors; @Data -public class LastRecordsCalculatedFieldState implements CalculatedFieldState { +public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { private Map> arguments; @@ -53,21 +55,16 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { } argumentValues.forEach((key, argumentEntry) -> { List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); - tsKvEntryList.addAll(argumentEntry.getKvEntries()); +// tsKvEntryList.addAll(argumentEntry.getKvEntries()); }); } @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration configuration = ctx.getConfiguration(); - Map configArguments = configuration.getArguments(); - Output output = configuration.getOutput(); - Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { - int limit = configArguments.get(key).getLimit(); + int limit = ctx.getArguments().get(key).getLimit(); List limitedEntries = entries.stream() .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) .limit(limit) @@ -79,13 +76,7 @@ public class LastRecordsCalculatedFieldState implements CalculatedFieldState { return limitedEntries; }); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return Futures.immediateFuture(calculatedFieldResult); + return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0e9b00ad7d..047bfd0f8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -35,16 +35,11 @@ import java.util.Map; @Data @Slf4j -public class ScriptCalculatedFieldState implements CalculatedFieldState { +public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @JsonIgnore private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - private Map arguments; - - public ScriptCalculatedFieldState() { - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -52,50 +47,35 @@ public class ScriptCalculatedFieldState implements CalculatedFieldState { @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } + public ListenableFuture performCalculation(CalculationContext ctx) { + if (isValid(this.arguments, ctx.getArguments())) { + if (calculatedFieldScriptEngine == null) { + initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); + } + ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - TbelInvokeService tbelInvokeService = ctx.getTbelInvokeService(); + return Futures.transform(resultFuture, result -> { + Map resultMap = result instanceof Map + ? JacksonUtil.convertValue(result, Map.class) + : new HashMap<>(); - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); + return buildResult(ctx.getOutput(), resultMap); + }, MoreExecutors.directExecutor()); } + return null; + } - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), calculatedFieldConfiguration, tbelInvokeService); + private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); } - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(arguments); - - return Futures.transform(resultFuture, result -> { - Output output = calculatedFieldConfiguration.getOutput(); - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - CalculatedFieldResult calculatedFieldResult = new CalculatedFieldResult(); - calculatedFieldResult.setType(output.getType()); - calculatedFieldResult.setScope(output.getScope()); - calculatedFieldResult.setResultMap(resultMap); - - return calculatedFieldResult; - }, MoreExecutors.directExecutor()); - } - - private void initEngine(TenantId tenantId, CalculatedFieldConfiguration calculatedFieldConfiguration, TbelInvokeService tbelInvokeService) { calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( tenantId, tbelInvokeService, - calculatedFieldConfiguration.getExpression(), - arguments.keySet().toArray(new String[0]) + expression, + this.arguments.keySet().toArray(new String[0]) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d3d3b5f636..fbdd1eb354 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -31,36 +31,17 @@ import java.util.HashMap; import java.util.Map; @Data -public class SimpleCalculatedFieldState implements CalculatedFieldState { - - private Map arguments; - - public SimpleCalculatedFieldState() { - } +public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new HashMap<>(); - } - argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - @Override public ListenableFuture performCalculation(CalculationContext ctx) { - CalculatedFieldConfiguration calculatedFieldConfiguration = ctx.getConfiguration(); - - Output output = calculatedFieldConfiguration.getOutput(); - Map arguments = calculatedFieldConfiguration.getArguments(); - - if (isValid(this.arguments, arguments)) { - CalculatedFieldResult result = new CalculatedFieldResult(); - String expression = calculatedFieldConfiguration.getExpression(); + if (isValid(this.arguments, ctx.getArguments())) { + String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); if (expr == null) { @@ -76,10 +57,8 @@ public class SimpleCalculatedFieldState implements CalculatedFieldState { double expressionResult = expr.evaluate(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(Map.of(output.getName(), expressionResult)); - return Futures.immediateFuture(result); + Output output = ctx.getOutput(); + return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); } return null; } From d684c8777a2d7ec438e8a4b7b7de0a6841eaae49 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 26 Nov 2024 17:33:01 +0200 Subject: [PATCH 041/438] improved usage of calculation ctx --- .../cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/CalculatedFieldResult.java | 8 +- ...efaultCalculatedFieldExecutionService.java | 95 +++++++++---------- ...Ctx.java => CalculatedFieldEntityCtx.java} | 8 +- ...d.java => CalculatedFieldEntityCtxId.java} | 2 +- .../service/cf/ctx/state/ArgumentEntry.java | 26 +++-- ...KvArgumentEntry.java => ArgumentType.java} | 15 +-- .../ctx/state/BaseCalculatedFieldState.java | 30 +++--- .../cf/ctx/state/CalculatedFieldCtx.java | 72 ++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 7 +- .../cf/ctx/state/CalculatedFieldState.java | 8 +- .../CalculatedFieldTbelScriptEngine.java | 18 +++- .../ctx/state/LastRecordsArgumentEntry.java | 16 +++- .../LastRecordsCalculatedFieldState.java | 46 +++++---- .../ctx/state/ScriptCalculatedFieldState.java | 47 ++------- .../ctx/state/SimpleCalculatedFieldState.java | 9 +- ...ext.java => SingleValueArgumentEntry.java} | 32 +++---- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../data/cf/configuration/Argument.java | 1 - .../BaseCalculatedFieldConfiguration.java | 4 - 20 files changed, 257 insertions(+), 191 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtx.java => CalculatedFieldEntityCtx.java} (79%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/{CalculatedFieldCtxId.java => CalculatedFieldEntityCtxId.java} (90%) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{KvArgumentEntry.java => ArgumentType.java} (72%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{CalculationContext.java => SingleValueArgumentEntry.java} (51%) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6b7b12655f..6d7cf0e741 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,6 +29,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); - void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 87f1d08a84..1f8a06c8fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -21,13 +21,15 @@ import org.thingsboard.server.common.data.AttributeScope; import java.util.Map; @Data -public class CalculatedFieldResult { +public final class CalculatedFieldResult { private String type; private AttributeScope scope; private Map resultMap; - public CalculatedFieldResult() { + public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + this.type = type; + this.scope = scope; + this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 322443c848..0d269f032f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,11 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtx; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.CalculationContext; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -115,7 +115,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -188,22 +189,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = cf.getEntityId(); calculatedFields.put(calculatedFieldId, cf); calculatedFieldLinks.put(calculatedFieldId, links); + CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(cf, entityId, callback); + initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE -> { log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); PageDataIterable assetIds = new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(cf, assetId, callback)); + assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } case DEVICE_PROFILE -> { log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(cf, deviceId, callback)); + deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -224,10 +227,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(calculatedFieldId, id -> calculatedFieldService.findById(tenantId, id)); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { + CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + return new CalculatedFieldCtx(calculatedField, tbelInvokeService); + }); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createArgumentEntry(entry.getValue()))); - updateOrInitializeState(calculatedField, calculatedField.getEntityId(), argumentValues); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); @@ -235,7 +241,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityTypeChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -246,15 +252,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) .stream() - .map(cfId -> calculatedFields.computeIfAbsent(cfId, id -> calculatedFieldService.findById(tenantId, id))) - .forEach(cf -> initializeStateForEntity(cf, entityId, callback)); + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -267,6 +273,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(newCalculatedField.getId(), callback); } else { calculatedFields.put(newCalculatedField.getId(), newCalculatedField); + calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -277,6 +284,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); + calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); List statesToRemove = states.keySet().stream() .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) @@ -309,20 +317,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldCtx.class))); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } - private void initializeStateForEntity(CalculatedField calculatedField, EntityId entityId, TbCallback callback) { - Map arguments = calculatedField.getConfiguration().getArguments(); + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedField, entityId, argument), new FutureCallback<>() { + arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { @Override public void onSuccess(ArgumentEntry result) { argumentValues.put(key, result); if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedField, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } } @@ -334,28 +342,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor)); } - private ListenableFuture fetchArgumentValue(CalculatedField calculatedField, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedField.getTenantId(); + private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedField.getType())) { + if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { return fetchLastRecords(tenantId, entityId, argument); } return fetchKvEntry(tenantId, entityId, argument); } private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { - long startTs = Math.max(argument.getStartTs(), 0); + long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long endTs = startTs + timeWindow; + long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, endTs, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createArgumentEntry, calculatedFieldExecutor); + return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { @@ -374,7 +382,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createArgumentEntry(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -389,32 +397,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedField calculatedField, EntityId entityId, Map argumentValues) { - CalculatedFieldCtxId ctxId = new CalculatedFieldCtxId(calculatedField.getUuidId(), entityId.getId()); - CalculatedFieldCtx calculatedFieldCtx = states.computeIfAbsent(ctxId, ctx -> new CalculatedFieldCtx(ctxId, null)); + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); - CalculatedFieldState state = calculatedFieldCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); if (state == null) { - state = createStateByType(calculatedField.getType()); + state = createStateByType(calculatedFieldCtx.getCfType()); } state.initState(argumentValues); - calculatedFieldCtx.setState(state); - states.put(ctxId, calculatedFieldCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(calculatedFieldCtx)); + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - ListenableFuture resultFuture = state.performCalculation(buildCalculationContext(calculatedField)); + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedField.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); } } @Override public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedField.getId(), entityId, t); + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } }, MoreExecutors.directExecutor()); @@ -433,17 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculationContext buildCalculationContext(CalculatedField calculatedField) { - CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); - return CalculationContext.builder() - .tenantId(calculatedField.getTenantId()) - .arguments(configuration.getArguments()) - .output(configuration.getOutput()) - .expression(configuration.getExpression()) - .tbelInvokeService(tbelInvokeService) - .build(); - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 4b2a6c918f..7a8384b6bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -19,15 +19,15 @@ import lombok.Data; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data -public class CalculatedFieldCtx { +public class CalculatedFieldEntityCtx { - private CalculatedFieldCtxId id; + private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldCtx() { + public CalculatedFieldEntityCtx() { } - public CalculatedFieldCtx(CalculatedFieldCtxId id, CalculatedFieldState state) { + public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index a316c54b76..f7c451efee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,5 +17,5 @@ package org.thingsboard.server.service.cf.ctx; import java.util.UUID; -public record CalculatedFieldCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 3097056d11..6218f38edf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,25 +15,35 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.stream.Collectors; +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "type" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), + @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") +}) public interface ArgumentEntry { + ArgumentType getType(); + Object getValue(); - static ArgumentEntry createArgumentEntry(KvEntry kvEntry) { - if (kvEntry instanceof TsKvEntry tsKvEntry) { - return new LastRecordsArgumentEntry(List.of(tsKvEntry)); - } else { - return new KvArgumentEntry(kvEntry); - } + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { + return new SingleValueArgumentEntry(kvEntry.getValue()); } - static ArgumentEntry createArgumentEntry(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries); + static ArgumentEntry createLastRecordsArgument(List kvEntries) { + return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index 0bd21d452e..f2f0eac60d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KvArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -15,17 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Data; -import org.thingsboard.server.common.data.kv.KvEntry; - -@Data -public class KvArgumentEntry implements ArgumentEntry { - - private final KvEntry kvEntry; - - @Override - public Object getValue() { - return kvEntry; - } - +public enum ArgumentType { + SINGLE_VALUE, LAST_RECORDS } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bac318a1b9..54df89e757 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,34 +15,36 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { - protected Map arguments; + protected Map arguments; public BaseCalculatedFieldState() { } + @Override + public Map getArguments() { + return this.arguments; + } + @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } -// argumentValues.forEach((key, value) -> arguments.put(key, value.getKvEntry())); - } - - protected CalculatedFieldResult buildResult(Output output, Map resultMap) { - CalculatedFieldResult result = new CalculatedFieldResult(); - result.setType(output.getType()); - result.setScope(output.getScope()); - result.setResultMap(resultMap); - return result; + arguments.putAll( + argumentValues.entrySet().stream() + .peek(entry -> { + if (entry.getValue() instanceof LastRecordsArgumentEntry) { + throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); + } + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java new file mode 100644 index 0000000000..10395a0d5d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.Data; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +@Data +public class CalculatedFieldCtx { + + private CalculatedFieldId cfId; + private TenantId tenantId; + private EntityId entityId; + private CalculatedFieldType cfType; + private Map arguments; + private Output output; + private String expression; + private TbelInvokeService tbelInvokeService; + private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.cfId = calculatedField.getId(); + this.tenantId = calculatedField.getTenantId(); + this.entityId = calculatedField.getEntityId(); + this.cfType = calculatedField.getType(); + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + this.arguments = configuration.getArguments(); + this.output = configuration.getOutput(); + this.expression = configuration.getExpression(); + this.tbelInvokeService = tbelInvokeService; + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + } + + private CalculatedFieldScriptEngine initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + return new CalculatedFieldTbelScriptEngine( + tenantId, + tbelInvokeService, + expression, + arguments.keySet().toArray(new String[0]) + ); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 6b54536019..212d971398 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -16,13 +16,16 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.kv.KvEntry; import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Map arguments); + + ListenableFuture> executeToMapAsync(Map arguments); + + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index e4c4440921..f0882cbea4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -41,12 +41,14 @@ public interface CalculatedFieldState { @JsonIgnore CalculatedFieldType getType(); - default boolean isValid(Map argumentValues, Map arguments) { - return argumentValues.keySet().containsAll(arguments.keySet()); + Map getArguments(); + + default boolean isValid(Map arguments) { + return getArguments().keySet().containsAll(arguments.keySet()); } void initState(Map argumentValues); - ListenableFuture performCalculation(CalculationContext ctx); + ListenableFuture performCalculation(CalculatedFieldCtx ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 5cc58b2a95..363eefd44e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -52,9 +53,9 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { + public ListenableFuture executeScriptAsync(Map arguments) { log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(KvEntry::getValue).toArray(); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -71,6 +72,19 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture> executeToMapAsync(Map arguments) { + return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture> executeToMapTransform(Object result) { + if (result instanceof Map) { + return Futures.immediateFuture((Map) result); + } + throw new IllegalArgumentException("Wrong result type: [" + result.getClass().getName() + "]"); + } + @Override public void destroy() { tbelInvokeService.release(this.scriptId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 8729f022fa..39da1838bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,19 +15,27 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import lombok.NoArgsConstructor; -import java.util.List; +import java.util.Map; @Data +@NoArgsConstructor +@AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private final List kvEntries; + private Map tsRecords; + + @Override + public ArgumentType getType() { + return ArgumentType.LAST_RECORDS; + } @Override public Object getValue() { - return kvEntries; + return tsRecords.values(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 0428d0823d..437707bfdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -15,14 +15,10 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import aj.org.objectweb.asm.TypeReference; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -37,8 +33,6 @@ import java.util.stream.Collectors; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - private Map> arguments; - public LastRecordsCalculatedFieldState() { } @@ -47,36 +41,46 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.LAST_RECORDS; } - @Override public void initState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } argumentValues.forEach((key, argumentEntry) -> { - List tsKvEntryList = arguments.computeIfAbsent(key, k -> new ArrayList<>()); -// tsKvEntryList.addAll(argumentEntry.getKvEntries()); + LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { + existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry + && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { + existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } }); } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, entries) -> { + arguments.replaceAll((key, argumentEntry) -> { int limit = ctx.getArguments().get(key).getLimit(); - List limitedEntries = entries.stream() - .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) - .limit(limit) - .collect(Collectors.toList()); - Map valueWithTs = limitedEntries.stream() - .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); - resultMap.put(key, valueWithTs); + // TODO: implement removing if size > limit + + +// List limitedEntries = entries.stream() +// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) +// .limit(limit) +// .collect(Collectors.toList()); +// +// Map valueWithTs = limitedEntries.stream() +// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); +// resultMap.put(key, valueWithTs); - return limitedEntries; +// return new LastRecordsArgumentEntry(limitedEntries); + return null; }); - return Futures.immediateFuture(buildResult(ctx.getOutput(), resultMap)); + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 047bfd0f8c..16905930d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -15,68 +15,37 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; @Data @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { - @JsonIgnore - private CalculatedFieldScriptEngine calculatedFieldScriptEngine; - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; } - @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { - if (calculatedFieldScriptEngine == null) { - initEngine(ctx.getTenantId(), ctx.getExpression(), ctx.getTbelInvokeService()); - } - - ListenableFuture resultFuture = calculatedFieldScriptEngine.executeScriptAsync(this.arguments); - - return Futures.transform(resultFuture, result -> { - Map resultMap = result instanceof Map - ? JacksonUtil.convertValue(result, Map.class) - : new HashMap<>(); - - return buildResult(ctx.getOutput(), resultMap); - }, MoreExecutors.directExecutor()); + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + Output output = ctx.getOutput(); + if (isValid(ctx.getArguments())) { + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } return null; } - private void initEngine(TenantId tenantId, String expression, TbelInvokeService tbelInvokeService) { - if (tbelInvokeService == null) { - throw new IllegalArgumentException("TBEL script engine is disabled!"); - } - - calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( - tenantId, - tbelInvokeService, - expression, - this.arguments.keySet().toArray(new String[0]) - ); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fbdd1eb354..64eb409d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -22,7 +22,6 @@ import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -39,8 +38,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { } @Override - public ListenableFuture performCalculation(CalculationContext ctx) { - if (isValid(this.arguments, ctx.getArguments())) { + public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { + if (isValid(ctx.getArguments())) { String expression = ctx.getExpression(); ThreadLocal customExpression = new ThreadLocal<>(); var expr = customExpression.get(); @@ -52,13 +51,13 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); Output output = ctx.getOutput(); - return Futures.immediateFuture(buildResult(output, Map.of(output.getName(), expressionResult))); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java similarity index 51% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index aaabaec0d4..504d213748 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculationContext.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,25 +15,25 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.Builder; +import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; - -import java.util.Map; +import lombok.NoArgsConstructor; @Data -@Builder -public class CalculationContext { +@NoArgsConstructor +@AllArgsConstructor +public class SingleValueArgumentEntry implements ArgumentEntry { + + private Object value; + + @Override + public ArgumentType getType() { + return ArgumentType.SINGLE_VALUE; + } - private TenantId tenantId; - private Map arguments; - private Output output; - private String expression; - private TbelInvokeService tbelInvokeService; + @Override + public Object getValue() { + return value; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3caaab6613..392e2637d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -685,7 +685,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityTypeChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index 0d70591a38..f34f5e9cb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -29,7 +29,6 @@ public class Argument { private String defaultValue; private int limit; - private long startTs; private long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f7cc53b7cf..f311fb737b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -102,7 +102,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("startTs", String.valueOf(argument.getStartTs())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); }); @@ -153,9 +152,6 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("limit")) { argument.setLimit(argumentNode.get("limit").asInt()); } - if (argumentNode.hasNonNull("startTs")) { - argument.setStartTs(argumentNode.get("startTs").asLong()); - } if (argumentNode.hasNonNull("timeWindow")) { argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); } From f9db64a14d036cbd6a5ffdbe3a733b177e9d081b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 27 Nov 2024 12:10:04 +0200 Subject: [PATCH 042/438] added deletion from map if ts out of timewindow --- .../service/cf/ctx/state/ArgumentEntry.java | 8 +++- .../ctx/state/LastRecordsArgumentEntry.java | 6 ++- .../LastRecordsCalculatedFieldState.java | 46 ++++++++----------- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++-- 5 files changed, 46 insertions(+), 38 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6218f38edf..6a62c0b1e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.TreeMap; import java.util.stream.Collectors; @JsonTypeInfo( @@ -34,16 +36,18 @@ import java.util.stream.Collectors; }) public interface ArgumentEntry { + @JsonIgnore ArgumentType getType(); Object getValue(); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { - return new SingleValueArgumentEntry(kvEntry.getValue()); + return new SingleValueArgumentEntry(kvEntry); } static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream() .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue))); + return new LastRecordsArgumentEntry(kvEntries.stream(). + collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 39da1838bc..93fabd3cb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -15,24 +15,26 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.Map; +import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor public class LastRecordsArgumentEntry implements ArgumentEntry { - private Map tsRecords; + private TreeMap tsRecords; @Override public ArgumentType getType() { return ArgumentType.LAST_RECORDS; } + @JsonIgnore @Override public Object getValue() { return tsRecords.values(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index 437707bfdb..dd69790236 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -19,16 +19,13 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.TreeMap; @Data public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @@ -44,16 +41,15 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public void initState(Map argumentValues) { if (arguments == null) { - arguments = new HashMap<>(); + arguments = new TreeMap<>(); } argumentValues.forEach((key, argumentEntry) -> { LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new HashMap<>())); + arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry - && singleValueArgumentEntry.getValue() instanceof TsKvEntry tsKvEntry) { - existingArgumentEntry.getTsRecords().put(tsKvEntry.getTs(), tsKvEntry.getValue()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); } }); } @@ -61,26 +57,22 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Map resultMap = new HashMap<>(); - arguments.replaceAll((key, argumentEntry) -> { - int limit = ctx.getArguments().get(key).getLimit(); - - // TODO: implement removing if size > limit - - -// List limitedEntries = entries.stream() -// .sorted(Comparator.comparingLong(TsKvEntry::getTs).reversed()) -// .limit(limit) -// .collect(Collectors.toList()); -// -// Map valueWithTs = limitedEntries.stream() -// .collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue)); -// resultMap.put(key, valueWithTs); - -// return new LastRecordsArgumentEntry(limitedEntries); - return null; + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); + tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); + resultMap.put(key, tsRecords); }); Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); } + private long calculateIntervalStart(long ts, long interval) { + return (ts / interval) * interval; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 64eb409d3a..fc97141806 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,9 +21,7 @@ import lombok.Data; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +49,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { customExpression.set(expr); } Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(((KvEntry) v.getValue()).getValueAsString()))); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); expr.setVariables(variables); double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 504d213748..e0db8c50fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,17 +15,29 @@ */ package org.thingsboard.server.service.cf.ctx.state; -import lombok.AllArgsConstructor; import lombok.Data; -import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; @Data -@NoArgsConstructor -@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + private long ts; private Object value; + public SingleValueArgumentEntry() { + } + + public SingleValueArgumentEntry(KvEntry entry) { + if (entry instanceof TsKvEntry) { + this.ts = ((TsKvEntry) entry).getTs(); + } else if (entry instanceof AttributeKvEntry) { + this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + } + this.value = entry.getValue(); + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; From 2080b439a7e207ff186202c615f4cc161c010076 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 28 Nov 2024 11:28:51 +0200 Subject: [PATCH 043/438] added entity added/deleted events handling --- .../cf/CalculatedFieldExecutionService.java | 4 ++ ...efaultCalculatedFieldExecutionService.java | 46 ++++++++++++-- .../entitiy/EntityStateSourcingListener.java | 6 +- .../queue/DefaultTbClusterService.java | 61 +++++++++++++++++-- .../queue/DefaultTbCoreConsumerService.java | 27 ++++++++ .../server/cluster/TbClusterService.java | 2 + common/proto/src/main/proto/queue.proto | 13 ++++ .../server/dao/asset/BaseAssetService.java | 2 +- 8 files changed, 147 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d7cf0e741..a1202ec74b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -31,4 +31,8 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); + + void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0d269f032f..3bb9be065d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -74,8 +74,8 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; -import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; @@ -227,6 +227,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { + log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); @@ -247,7 +248,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) @@ -257,10 +257,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); }); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, newProfileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + @Override + public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); + log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } catch (Exception e) { + log.trace("Failed to process entity type update msg: [{}]", proto, e); + } + } + + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + + @Override + public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 154f5f4833..57293169a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -146,7 +146,11 @@ public class EntityStateSourcingListener { log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { - case ASSET, ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET -> { + Asset asset = (Asset) event.getEntity(); + tbClusterService.onAssetDeleted(tenantId, asset, null); + } + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 58a16d9c0a..78a1f27a55 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -388,11 +388,26 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); + handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); } + @Override + public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { + AssetId assetId = asset.getId(); + handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); + } + + private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); + if (cfExistsByProfile) { + sendEntityDeletedEvent(tenantId, entityId); + } + } + @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -613,11 +628,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileChange(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } + } else { + handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -631,16 +648,26 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileChange(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } + } else { + handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileChange(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); + boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); + if (cfExistsByOldProfile || cfExistsByNewProfile) { + sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + } + } + + private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { + boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityTypeUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); + sendEntityAddedEvent(tenantId, entityId, profileId); } } @@ -801,7 +828,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityTypeUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -817,4 +844,26 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } + private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { + TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + builder.setEntityProfileType(profileId.getEntityType().name()); + builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); + builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + TransportProtos.EntityAddMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); + } + + private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { + TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); + builder.setEntityType(entityId.getEntityType().name()); + builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 392e2637d1..d477c6d753 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,6 +320,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { + var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 131b50a52c..69334b774f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -100,6 +100,8 @@ public interface TbClusterService extends TbQueueClusterService { void onAssetUpdated(Asset asset, Asset old); + void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback); + void onResourceChange(TbResourceInfo resource, TbQueueCallback callback); void onResourceDeleted(TbResourceInfo resource, TbQueueCallback callback); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index bf49a68c80..b1ca9b5e56 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,6 +794,17 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } +message EntityAddMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + string entityProfileType = 6; + int64 profileIdMSB = 7; + int64 profileIdLSB = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1527,6 +1538,8 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; + EntityAddMsgProto entityAddMsg = 55; + EntityDeleteMsg entityDeleteMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 7242cfde68..564da127ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -239,7 +239,7 @@ public class BaseAssetService extends AbstractCachedEntityService Date: Thu, 28 Nov 2024 16:07:55 +0200 Subject: [PATCH 044/438] implemented update telemetry for entity by profile --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 61 +++++++++++++------ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index a1202ec74b..b605ee909e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 3bb9be065d..d821f632fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -225,7 +225,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { @@ -234,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, calculatedFieldCtx.getEntityId(), argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 76c1383f6a..f6c19eb078 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -56,6 +58,8 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -85,6 +89,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbApiUsageStateService apiUsageStateService; private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private ExecutorService tsCallBackExecutor; @@ -97,7 +103,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldExecutionService calculatedFieldExecutionService, + TbAssetProfileCache assetProfileCache, + TbDeviceProfileCache deviceProfileCache) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; @@ -105,6 +113,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer this.apiUsageStateService = apiUsageStateService; this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.assetProfileCache = assetProfileCache; + this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,8 +211,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - List cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { + EntityId profileId; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + } + List cfLinks = new ArrayList<>(); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); @@ -211,34 +230,36 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer Map updatedTelemetry = telemetry.stream() .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - }, + entry -> getMappedKey(entry, attributes, timeSeries), entry -> entry, (v1, v2) -> v1 )); if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, calculatedFieldId, updatedTelemetry); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); } }); } } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), From b910760b05c8597b1e44853f4eb928c7560b0865 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 12:29:41 +0200 Subject: [PATCH 045/438] improved fetching cfs and states from db --- ...efaultCalculatedFieldExecutionService.java | 51 +++++++++---------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d821f632fe..2147e0a1b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,7 +36,6 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; @@ -81,7 +80,6 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -114,7 +112,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -131,7 +128,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); } @PreDestroy @@ -176,7 +172,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); + CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -184,11 +180,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } } - List links = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cf != null) { EntityId entityId = cf.getEntityId(); - calculatedFields.put(calculatedFieldId, cf); - calculatedFieldLinks.put(calculatedFieldId, links); CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); switch (entityId.getEntityType()) { @@ -229,7 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = calculatedFields.computeIfAbsent(id, cfId -> calculatedFieldService.findById(tenantId, id)); + CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); return new CalculatedFieldCtx(calculatedField, tbelInvokeService); }); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -300,14 +293,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private boolean onCalculatedFieldUpdate(CalculatedField newCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFields.get(newCalculatedField.getId()); + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, newCalculatedField)) { - onCalculatedFieldDelete(newCalculatedField.getId(), callback); + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(newCalculatedField.getId(), newCalculatedField); - calculatedFieldsCtx.put(newCalculatedField.getId(), new CalculatedFieldCtx(newCalculatedField, tbelInvokeService)); + calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); + calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -316,12 +309,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { - calculatedFieldLinks.remove(calculatedFieldId); calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() - .filter(ctxId -> !calculatedFields.containsKey(new CalculatedFieldId(ctxId.cfId()))) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -331,6 +323,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -346,15 +342,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); @@ -433,7 +420,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctx -> new CalculatedFieldEntityCtx(entityCtxId, null)); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -462,6 +449,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, null); + } + return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + } + private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); From 894f4ead1b120d46809e6d03c326fa195fd69b0e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 29 Nov 2024 17:31:56 +0200 Subject: [PATCH 046/438] implemented logic not to fetch tenant/customer telemetry for each field --- ...efaultCalculatedFieldExecutionService.java | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2147e0a1b7..732d050a22 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfig import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -80,6 +81,7 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -115,6 +117,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); + private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); + private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -366,6 +371,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); + if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { + return fetchFromStorage(tenantId, argumentEntityId, argument); + } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -375,6 +383,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } + private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { + List kvEntries; + if (EntityType.TENANT.equals(entityId.getEntityType())) { + kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + } else { + kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + } + for (KvEntry kvEntry : kvEntries) { + if (kvEntry.getKey().equals(argument.getKey())) { + return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); + } + } + return fetchKvEntry(tenantId, entityId, argument); + } + private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -403,7 +426,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, kvEntry); + } + return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); + }, calculatedFieldExecutor); + } + + private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { + if (kvEntry.isEmpty()) { + return; + } + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { + kvEntries.add(kvEntry.get()); + } } private KvEntry createDefaultKvEntry(Argument argument) { From 19234df2b8e03d8d2af392faa32dfb44cc71bc77 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 5 Dec 2024 16:46:12 +0200 Subject: [PATCH 047/438] improved implementation of telemetry update --- ...efaultCalculatedFieldExecutionService.java | 60 ++++++++++++------- .../DefaultTelemetrySubscriptionService.java | 11 ++-- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 732d050a22..4a1e987687 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -177,7 +177,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOfFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -226,14 +226,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { try { log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> { - CalculatedField calculatedField = getOfFetchFromDb(tenantId, id); - return new CalculatedFieldCtx(calculatedField, tbelInvokeService); - }); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - log.info("Successfully updated time series for calculatedFieldId: [{}]", calculatedFieldId); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> { + if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { + updateStorage(tenantId, entityId, Optional.of(entry.getValue())); + } + return ArgumentEntry.createSingleValueArgument(entry.getValue()); + })); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE + ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) + : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); + entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); } @@ -299,7 +317,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOfFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -328,7 +346,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOfFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } @@ -435,17 +453,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - if (kvEntry.isEmpty()) { - return; - } - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null && !kvEntries.contains(kvEntry.get())) { - kvEntries.add(kvEntry.get()); - } + kvEntry.ifPresent(entry -> { + List kvEntries = switch (entityId.getEntityType()) { + case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); + case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); + default -> null; + }; + if (kvEntries != null) { + kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); + kvEntries.add(entry); + } + }); } private KvEntry createDefaultKvEntry(Argument argument) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f6c19eb078..ad64204e0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -212,16 +212,15 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType)) { - EntityId profileId; + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; if (EntityType.ASSET.equals(entityType)) { profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else { + } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId)); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); if (!cfLinks.isEmpty()) { cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); From dfeb0a7dbaf30b28529e27449f0fac9015f066f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 6 Dec 2024 11:11:24 +0200 Subject: [PATCH 048/438] made executeScrioptAsync method flexible --- .../cf/ctx/state/CalculatedFieldScriptEngine.java | 4 ++-- .../cf/ctx/state/CalculatedFieldTbelScriptEngine.java | 11 ++++------- .../cf/ctx/state/ScriptCalculatedFieldState.java | 3 ++- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 212d971398..779f52c5d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -21,9 +21,9 @@ import java.util.Map; public interface CalculatedFieldScriptEngine { - ListenableFuture executeScriptAsync(Map arguments); + ListenableFuture executeScriptAsync(Object[] args); - ListenableFuture> executeToMapAsync(Map arguments); + ListenableFuture> executeToMapAsync(Object[] args); ListenableFuture> executeToMapTransform(Object result); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 363eefd44e..7ac032573f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,11 +19,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import javax.script.ScriptException; import java.util.Map; @@ -53,9 +51,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture executeScriptAsync(Map arguments) { - log.trace("execute script async, arguments {}", arguments); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + public ListenableFuture executeScriptAsync(Object[] args) { + log.trace("Executing script async, args {}", args); return Futures.transformAsync(tbelInvokeService.invokeScript(tenantId, null, this.scriptId, args), o -> { try { @@ -73,8 +70,8 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng } @Override - public ListenableFuture> executeToMapAsync(Map arguments) { - return Futures.transformAsync(executeScriptAsync(arguments), this::executeToMapTransform, MoreExecutors.directExecutor()); + public ListenableFuture> executeToMapAsync(Object[] args) { + return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 16905930d3..99befa6e65 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,7 +39,8 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(this.arguments); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From 303851015035cc3715190b5fc708ad6da173e807 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 11:04:05 +0200 Subject: [PATCH 049/438] added implementation for handling profile entities --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 96 ++++++++++--------- .../queue/DefaultTbClusterService.java | 22 ++--- .../queue/DefaultTbCoreConsumerService.java | 27 ++---- common/proto/src/main/proto/queue.proto | 7 +- 5 files changed, 71 insertions(+), 85 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index b605ee909e..5a85529f6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -32,8 +32,6 @@ public interface CalculatedFieldExecutionService { void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); - void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback); - - void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback); + void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 4a1e987687..7ded80d013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,11 +39,9 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -83,6 +81,7 @@ import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -117,6 +116,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); @@ -194,17 +194,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } - case ASSET_PROFILE -> { - log.info("Initializing state for all assets in profile: tenantId=[{}], assetProfileId=[{}]", tenantId, entityId); - PageDataIterable assetIds = new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) entityId, pageLink), initFetchPackSize); - assetIds.forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); - } - case DEVICE_PROFILE -> { - log.info("Initializing state for all devices in profile: tenantId=[{}], deviceProfileId=[{}]", tenantId, entityId); - PageDataIterable deviceIds = new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityId, pageLink), initFetchPackSize); - deviceIds.forEach(deviceId -> initializeStateForEntity(calculatedFieldCtx, deviceId, callback)); + case ASSET_PROFILE, DEVICE_PROFILE -> { + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -241,10 +233,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - PageDataIterable entities = cfEntityId.getEntityType() == EntityType.ASSET_PROFILE - ? new PageDataIterable<>(pageLink -> assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) cfEntityId, pageLink), initFetchPackSize) - : new PageDataIterable<>(pageLink -> deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) cfEntityId, pageLink), initFetchPackSize); - entities.forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); } @@ -266,6 +255,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(oldProfileId).remove(entityId); + profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -280,39 +272,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityAdded(TransportProtos.EntityAddMsgProto proto, TbCallback callback) { + public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received EntityCreateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - @Override - public void onEntityDeleted(TransportProtos.EntityDeleteMsg proto, TbCallback callback) { - try { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received EntityDeleteMsg for processing: entityId=[{}]", entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.get(profileId).remove(entityId); + List statesToRemove = states.keySet().stream() + .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); + rocksDBService.deleteAll(statesToRemove); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + } } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); + log.trace("Failed to process profile entity msg: [{}]", proto, e); } } @@ -350,6 +331,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); } + private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { + return switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -365,6 +364,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } + private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) + .stream() + .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); + } + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { Map arguments = calculatedFieldCtx.getArguments(); Map argumentValues = new HashMap<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 78a1f27a55..a706f943c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -404,7 +404,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityDeletedEvent(tenantId, entityId); + sendProfileEntityEvent(tenantId, entityId, profileId, false, true); } } @@ -667,7 +667,7 @@ public class DefaultTbClusterService implements TbClusterService { private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); if (cfExistsByProfile) { - sendEntityAddedEvent(tenantId, entityId, profileId); + sendProfileEntityEvent(tenantId, entityId, profileId, true, false); } } @@ -844,8 +844,8 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendEntityAddedEvent(TenantId tenantId, EntityId entityId, EntityId profileId) { - TransportProtos.EntityAddMsgProto.Builder builder = TransportProtos.EntityAddMsgProto.newBuilder(); + private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -854,16 +854,10 @@ public class DefaultTbClusterService implements TbClusterService { builder.setEntityProfileType(profileId.getEntityType().name()); builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); - TransportProtos.EntityAddMsgProto msg = builder.build(); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityAddMsg(msg).build(), null); - } - - private void sendEntityDeletedEvent(TenantId tenantId, EntityId entityId) { - TransportProtos.EntityDeleteMsg.Builder builder = TransportProtos.EntityDeleteMsg.newBuilder(); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityDeleteMsg(builder).build(), null); + builder.setAdded(added); + builder.setDeleted(deleted); + TransportProtos.ProfileEntityMsgProto msg = builder.build(); + pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d477c6d753..6cf42cb893 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -320,10 +320,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityAdded(entityCreateMsg, callback)); + private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { + var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process entity create message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityDeleteMsg entityDeleteMsg, TbCallback callback) { - var entityId = EntityIdFactory.getByTypeAndUuid(entityDeleteMsg.getEntityType(), new UUID(entityDeleteMsg.getEntityIdMSB(), entityDeleteMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityDeleted(entityDeleteMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("Failed to process entity delete message for entity [{}]", entityId, t); + log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index b1ca9b5e56..76da257b32 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -794,7 +794,7 @@ message EntityProfileUpdateMsgProto { int64 newProfileIdLSB = 10; } -message EntityAddMsgProto { +message ProfileEntityMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -803,6 +803,8 @@ message EntityAddMsgProto { string entityProfileType = 6; int64 profileIdMSB = 7; int64 profileIdLSB = 8; + bool added = 9; + bool deleted = 10; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. @@ -1538,8 +1540,7 @@ message ToCoreMsg { DeviceInactivityProto deviceInactivityMsg = 52; CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - EntityAddMsgProto entityAddMsg = 55; - EntityDeleteMsg entityDeleteMsg = 56; + ProfileEntityMsgProto profileEntityMsg = 55; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 4fed2cd416b13b7f05159ba893f47dcd788fb13d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 9 Dec 2024 13:04:41 +0200 Subject: [PATCH 050/438] improved ability to fetch common arguments for profile cfs --- ...efaultCalculatedFieldExecutionService.java | 120 +++++++++--------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7ded80d013..f2629381b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -41,7 +41,6 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -89,7 +88,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -117,8 +116,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final ConcurrentMap states = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final ConcurrentMap> tenantStorage = new ConcurrentHashMap<>(); - private final ConcurrentMap> customerStorage = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -196,7 +193,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> initializeStateForEntity(calculatedFieldCtx, assetId, callback)); + fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { + initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + }); + }); } default -> throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); @@ -221,12 +222,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, Optional.of(entry.getValue())); - } - return ArgumentEntry.createSingleValueArgument(entry.getValue()); - })); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); EntityId cfEntityId = calculatedField.getEntityId(); switch (cfEntityId.getEntityType()) { @@ -371,33 +367,71 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - Map arguments = calculatedFieldCtx.getArguments(); + private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { Map argumentValues = new HashMap<>(); - AtomicInteger remaining = new AtomicInteger(arguments.size()); - arguments.forEach((key, argument) -> Futures.addCallback(fetchArgumentValue(calculatedFieldCtx, entityId, argument), new FutureCallback<>() { + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && + !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { + futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override - public void onSuccess(ArgumentEntry result) { - argumentValues.put(key, result); - if (remaining.decrementAndGet() == 0) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + public void onSuccess(List results) { + onComplete.accept(argumentValues); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to fetch common arguments", t); + callback.onFailure(t); + } + }, calculatedFieldCallbackExecutor); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { + initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); + } + + private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { + Map argumentValues = new HashMap<>(commonArguments); + List> futures = new ArrayList<>(); + + calculatedFieldCtx.getArguments().forEach((key, argument) -> { + if (!commonArguments.containsKey(key)) { + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + } + }); + + Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { + @Override + public void onSuccess(List results) { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + callback.onSuccess(); } @Override public void onFailure(Throwable t) { - log.warn("Failed to initialize state for entity: [{}]", entityId, t); + log.error("Failed to initialize state for entity: [{}]", entityId, t); callback.onFailure(t); } - }, calculatedFieldCallbackExecutor)); + }, calculatedFieldCallbackExecutor); } private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { TenantId tenantId = calculatedFieldCtx.getTenantId(); EntityId argumentEntityId = argument.getEntityId(); - if (EntityType.TENANT.equals(argumentEntityId.getEntityType()) || EntityType.CUSTOMER.equals(argumentEntityId.getEntityType())) { - return fetchFromStorage(tenantId, argumentEntityId, argument); - } EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; @@ -407,21 +441,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchFromStorage(TenantId tenantId, EntityId entityId, Argument argument) { - List kvEntries; - if (EntityType.TENANT.equals(entityId.getEntityType())) { - kvEntries = tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - } else { - kvEntries = customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - } - for (KvEntry kvEntry : kvEntries) { - if (kvEntry.getKey().equals(argument.getKey())) { - return Futures.immediateFuture(ArgumentEntry.createSingleValueArgument(kvEntry)); - } - } - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); @@ -450,26 +469,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldExecutor); default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; - return Futures.transform(kvEntryFuture, kvEntry -> { - if (EntityType.TENANT.equals(entityId.getEntityType()) || EntityType.CUSTOMER.equals(entityId.getEntityType())) { - updateStorage(tenantId, entityId, kvEntry); - } - return ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)); - }, calculatedFieldExecutor); - } - - private void updateStorage(TenantId tenantId, EntityId entityId, Optional kvEntry) { - kvEntry.ifPresent(entry -> { - List kvEntries = switch (entityId.getEntityType()) { - case TENANT -> tenantStorage.computeIfAbsent(tenantId, id -> new ArrayList<>()); - case CUSTOMER -> customerStorage.computeIfAbsent((CustomerId) entityId, id -> new ArrayList<>()); - default -> null; - }; - if (kvEntries != null) { - kvEntries.removeIf(existingEntry -> existingEntry.getKey().equals(entry.getKey())); - kvEntries.add(entry); - } - }); + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { From 86569c312e96e297ad1de55b31c681bf32c36a28 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 10 Dec 2024 12:21:13 +0200 Subject: [PATCH 051/438] added ability to perform calculations on the last records --- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../ctx/state/LastRecordsArgumentEntry.java | 2 +- .../LastRecordsCalculatedFieldState.java | 37 ++++++++++--------- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 10395a0d5d..b436e0421e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -51,7 +51,7 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { + if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java index 93fabd3cb8..8b672d4de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java @@ -37,7 +37,7 @@ public class LastRecordsArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords.values(); + return tsRecords; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java index dd69790236..fbcf72fda9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java @@ -17,13 +17,13 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; import java.util.Map; import java.util.TreeMap; @@ -56,23 +56,24 @@ public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Map resultMap = new HashMap<>(); - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - long necessaryIntervalTs = calculateIntervalStart(System.currentTimeMillis(), argument.getTimeWindow()); - tsRecords.entrySet().removeIf(tsRecord -> calculateIntervalStart(tsRecord.getKey(), argument.getTimeWindow()) < necessaryIntervalTs); - resultMap.put(key, tsRecords); - }); - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), resultMap)); - } - - private long calculateIntervalStart(long ts, long interval) { - return (ts / interval) * interval; + if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); + } + return null; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 99befa6e65..ba050d3b71 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -37,10 +37,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - Output output = ctx.getOutput(); if (isValid(ctx.getArguments())) { Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); return Futures.transform(resultFuture, result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), MoreExecutors.directExecutor() From a73affea23683c902a5ee45ad3b569c0463fd41b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 12 Dec 2024 17:04:24 +0200 Subject: [PATCH 052/438] removed CF type LAST_RECORDS and implemented this functionality as a script(added new argument type TS_ROLLING) --- ...efaultCalculatedFieldExecutionService.java | 49 ++++++------ .../service/cf/ctx/state/ArgumentEntry.java | 6 +- .../service/cf/ctx/state/ArgumentType.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 26 +++--- .../cf/ctx/state/CalculatedFieldState.java | 2 - .../LastRecordsCalculatedFieldState.java | 79 ------------------- .../ctx/state/ScriptCalculatedFieldState.java | 14 +++- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- ...Entry.java => TsRollingArgumentEntry.java} | 4 +- .../common/data/cf/CalculatedFieldType.java | 2 +- .../BaseCalculatedFieldConfiguration.java | 4 +- .../CalculatedFieldConfiguration.java | 3 +- ...stRecordsCalculatedFieldConfiguration.java | 39 --------- .../dao/model/sql/CalculatedFieldEntity.java | 2 - ...efaultNativeCalculatedFieldRepository.java | 2 - 15 files changed, 64 insertions(+), 172 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{LastRecordsArgumentEntry.java => TsRollingArgumentEntry.java} (91%) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2629381b2..050c2d7e73 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -73,12 +73,12 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import org.thingsboard.server.service.cf.ctx.state.LastRecordsCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -435,41 +435,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) ? targetEntityId : argumentEntityId; - if (CalculatedFieldType.LAST_RECORDS.equals(calculatedFieldCtx.getCfType())) { - return fetchLastRecords(tenantId, entityId, argument); - } return fetchKvEntry(tenantId, entityId, argument); } - private ListenableFuture fetchLastRecords(TenantId tenantId, EntityId entityId, Argument argument) { + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> lastRecordsFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(lastRecordsFuture, ArgumentEntry::createLastRecordsArgument, calculatedFieldExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - ListenableFuture> kvEntryFuture = switch (argument.getType()) { - case "ATTRIBUTES" -> Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - MoreExecutors.directExecutor()); - case "TIME_SERIES" -> Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of( - new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)) - )), - calculatedFieldExecutor); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldExecutor); + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -547,7 +547,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return switch (calculatedFieldType) { case SIMPLE -> new SimpleCalculatedFieldState(); case SCRIPT -> new ScriptCalculatedFieldState(); - case LAST_RECORDS -> new LastRecordsCalculatedFieldState(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 6a62c0b1e3..f70d614123 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SingleValueArgumentEntry.class, name = "SINGLE_VALUE"), - @JsonSubTypes.Type(value = LastRecordsArgumentEntry.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = TsRollingArgumentEntry.class, name = "TS_ROLLING") }) public interface ArgumentEntry { @@ -45,8 +45,8 @@ public interface ArgumentEntry { return new SingleValueArgumentEntry(kvEntry); } - static ArgumentEntry createLastRecordsArgument(List kvEntries) { - return new LastRecordsArgumentEntry(kvEntries.stream(). + static ArgumentEntry createTsRollingArgument(List kvEntries) { + return new TsRollingArgumentEntry(kvEntries.stream(). collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java index f2f0eac60d..360529a7e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java @@ -16,5 +16,5 @@ package org.thingsboard.server.service.cf.ctx.state; public enum ArgumentType { - SINGLE_VALUE, LAST_RECORDS + SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 54df89e757..59b007a420 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -36,15 +35,22 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - arguments.putAll( - argumentValues.entrySet().stream() - .peek(entry -> { - if (entry.getValue() instanceof LastRecordsArgumentEntry) { - throw new IllegalArgumentException("Last records argument entry is not allowed for single calculated field state"); - } - }) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) - ); + argumentValues.forEach((key, argumentEntry) -> { + ArgumentEntry existingArgumentEntry = arguments.get(key); + if (existingArgumentEntry != null) { + if (existingArgumentEntry instanceof SingleValueArgumentEntry) { + arguments.put(key, argumentEntry); + } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + } + } + } else { + arguments.put(key, argumentEntry); + } + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f0882cbea4..a5ac6b2c47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -34,7 +33,6 @@ import java.util.Map; @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldState.class, name = "SIMPLE"), @JsonSubTypes.Type(value = ScriptCalculatedFieldState.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldState.class, name = "LAST_RECORDS") }) public interface CalculatedFieldState { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java deleted file mode 100644 index fbcf72fda9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsCalculatedFieldState.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.ctx.state; - -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.Output; -import org.thingsboard.server.service.cf.CalculatedFieldResult; - -import java.util.Map; -import java.util.TreeMap; - -@Data -public class LastRecordsCalculatedFieldState extends BaseCalculatedFieldState { - - public LastRecordsCalculatedFieldState() { - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } - - @Override - public void initState(Map argumentValues) { - if (arguments == null) { - arguments = new TreeMap<>(); - } - argumentValues.forEach((key, argumentEntry) -> { - LastRecordsArgumentEntry existingArgumentEntry = (LastRecordsArgumentEntry) - arguments.computeIfAbsent(key, k -> new LastRecordsArgumentEntry(new TreeMap<>())); - if (argumentEntry instanceof LastRecordsArgumentEntry lastRecordsArgumentEntry) { - existingArgumentEntry.getTsRecords().putAll(lastRecordsArgumentEntry.getTsRecords()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - } - }); - } - - @Override - public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((LastRecordsArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return null; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index ba050d3b71..b9b98f9c5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; +import java.util.TreeMap; @Data @Slf4j @@ -38,6 +40,16 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { if (isValid(ctx.getArguments())) { + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); + } + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); @@ -46,7 +58,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { MoreExecutors.directExecutor() ); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index fc97141806..491419b40a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -57,7 +57,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { Output output = ctx.getOutput(); return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } - return null; + return Futures.immediateFuture(null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 8b672d4de9..1166da113d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/LastRecordsArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,13 +25,13 @@ import java.util.TreeMap; @Data @NoArgsConstructor @AllArgsConstructor -public class LastRecordsArgumentEntry implements ArgumentEntry { +public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords; @Override public ArgumentType getType() { - return ArgumentType.LAST_RECORDS; + return ArgumentType.TS_ROLLING; } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java index 63b6d8d1dd..89173b35b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldType.java @@ -17,6 +17,6 @@ package org.thingsboard.server.common.data.cf; public enum CalculatedFieldType { - SIMPLE, SCRIPT, LAST_RECORDS + SIMPLE, SCRIPT } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index f311fb737b..ac36991a61 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -69,10 +69,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel Argument argument = entry.getValue(); if (argument.getEntityId().equals(entityId)) { switch (argument.getType()) { - case "ATTRIBUTES": + case "ATTRIBUTE": linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); break; - case "TIME_SERIES": + case "TS_LATEST", "TS_ROLLING": linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); break; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 15f7a82c40..5c428bd628 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -35,8 +35,7 @@ import java.util.UUID; ) @JsonSubTypes({ @JsonSubTypes.Type(value = SimpleCalculatedFieldConfiguration.class, name = "SIMPLE"), - @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT"), - @JsonSubTypes.Type(value = LastRecordsCalculatedFieldConfiguration.class, name = "LAST_RECORDS") + @JsonSubTypes.Type(value = ScriptCalculatedFieldConfiguration.class, name = "SCRIPT") }) public interface CalculatedFieldConfiguration { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java deleted file mode 100644 index c3f5804227..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/LastRecordsCalculatedFieldConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.cf.configuration; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; - -import java.util.UUID; - -@Data -public class LastRecordsCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - - public LastRecordsCalculatedFieldConfiguration() { - } - - public LastRecordsCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - - @Override - public CalculatedFieldType getType() { - return CalculatedFieldType.LAST_RECORDS; - } -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index b06676f70b..6aaaf05836 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -124,7 +123,6 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return switch (CalculatedFieldType.valueOf(type)) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 2acd4d75c6..a5a2743f26 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.LastRecordsCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -140,7 +139,6 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF return switch (type) { case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - case LAST_RECORDS -> new LastRecordsCalculatedFieldConfiguration(config, entityType, entityId); }; } From 4668bece03229c34aa74c335bd6602ddf9885c71 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 13 Dec 2024 11:14:56 +0200 Subject: [PATCH 053/438] fixed tests --- .../server/controller/CalculatedFieldControllerTest.java | 4 ++-- .../org/thingsboard/server/dao/service/AssetServiceTest.java | 4 ++-- .../server/dao/service/CalculatedFieldServiceTest.java | 4 ++-- .../thingsboard/server/dao/service/CustomerServiceTest.java | 4 ++-- .../org/thingsboard/server/dao/service/DeviceServiceTest.java | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 77ca268d12..ba1dfb1fec 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -140,7 +140,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +149,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 87f2cb1f45..b0870f3dc5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -884,7 +884,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +893,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 77ed026b1d..9a1719e715 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -153,7 +153,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +162,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 94c8440057..6671e0e821 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -379,7 +379,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +388,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index d5394a3494..f2f8686bc3 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1222,7 +1222,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TIME_SERIES"); + argument.setType("TS_LATEST"); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1231,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TIME_SERIES"); + output.setType("TS_LATEST"); config.setOutput(output); From 19f6f323260347e5af7eb0317dbe168dd9cb8e52 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 16 Dec 2024 12:47:55 +0200 Subject: [PATCH 054/438] moved logic when telemetry update to main cf service --- .../cf/CalculatedFieldExecutionService.java | 5 +- .../service/cf/CalculatedFieldResult.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 149 +++++++++++++----- .../cf/ctx/CalculatedFieldEntityCtx.java | 5 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 + .../ctx/state/BaseCalculatedFieldState.java | 22 ++- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- .../cf/ctx/state/CalculatedFieldState.java | 7 +- .../ctx/state/ScriptCalculatedFieldState.java | 35 ++-- .../ctx/state/SimpleCalculatedFieldState.java | 37 ++--- .../ctx/state/SingleValueArgumentEntry.java | 9 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 + .../DefaultTelemetrySubscriptionService.java | 78 +-------- 13 files changed, 186 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 5a85529f6b..302e4b2511 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,19 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.Map; +import java.util.List; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 1f8a06c8fa..e8ea318bf6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -23,9 +23,9 @@ import java.util.Map; @Data public final class CalculatedFieldResult { - private String type; - private AttributeScope scope; - private Map resultMap; + private final String type; + private final AttributeScope scope; + private final Map resultMap; public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { this.type = type; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 050c2d7e73..c1a86ac36f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,16 +36,20 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -76,6 +80,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; @@ -102,6 +108,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -112,6 +120,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); @@ -130,6 +139,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(this::fetchCalculatedFields); + } + + private void fetchCalculatedFields() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); + states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy @@ -216,30 +235,77 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { try { - log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); - } + EntityType entityType = entityId.getEntityType(); + if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { + EntityId profileId = null; + if (EntityType.ASSET.equals(entityType)) { + profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + } else if (EntityType.DEVICE.equals(entityType)) { + profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + cfLinks.forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + Map attributes = link.getConfiguration().getAttributes(); + Map timeSeries = link.getConfiguration().getTimeSeries(); + Map updatedTelemetry = telemetry.stream() + .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, attributes, timeSeries), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + } + }); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } catch (Exception e) { - log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); + log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + } + } + + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); + CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + Map argumentValues = updatedTelemetry.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + + EntityId cfEntityId = calculatedField.getEntityId(); + switch (cfEntityId.getEntityType()) { + case ASSET_PROFILE, DEVICE_PROFILE -> { + boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); + if (isCommonEntity) { + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + } else { + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + } + } + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + } + log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); + } + + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } + return entry.getKey(); } @Override @@ -493,26 +559,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (state == null) { state = createStateByType(calculatedFieldCtx.getCfType()); } - state.initState(argumentValues); - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); - } - } + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + + boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); + if (allArgsPresent) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); + } + }, MoreExecutors.directExecutor()); } - }, MoreExecutors.directExecutor()); - + } } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { @@ -520,14 +589,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (stateStr == null) { return new CalculatedFieldEntityCtx(entityCtxId, null); } - return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class); + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java index 7a8384b6bf..e6dc021951 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java @@ -16,17 +16,16 @@ package org.thingsboard.server.service.cf.ctx; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @Data +@NoArgsConstructor public class CalculatedFieldEntityCtx { private CalculatedFieldEntityCtxId id; private CalculatedFieldState state; - public CalculatedFieldEntityCtx() { - } - public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { this.id = id; this.state = state; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index f70d614123..78222244c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -41,6 +41,8 @@ public interface ArgumentEntry { Object getValue(); + boolean hasUpdatedValue(ArgumentEntry entry); + static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 59b007a420..ae6fc9033a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -31,26 +32,39 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } @Override - public void initState(Map argumentValues) { + public boolean updateState(Map argumentValues) { if (arguments == null) { arguments = new HashMap<>(); } + AtomicBoolean stateUpdated = new AtomicBoolean(false); argumentValues.forEach((key, argumentEntry) -> { ArgumentEntry existingArgumentEntry = arguments.get(key); if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - arguments.put(key, argumentEntry); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + arguments.put(key, argumentEntry); + stateUpdated.set(true); + } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + stateUpdated.set(true); + } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { + existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + stateUpdated.set(true); + } + } } } else { arguments.put(key, argumentEntry); + stateUpdated.set(true); } }); + return stateUpdated.get(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index b436e0421e..2cd5c68144 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -35,7 +35,7 @@ public class CalculatedFieldCtx { private TenantId tenantId; private EntityId entityId; private CalculatedFieldType cfType; - private Map arguments; + private final Map arguments; private Output output; private String expression; private TbelInvokeService tbelInvokeService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index a5ac6b2c47..3c4a680df1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.Map; @@ -41,11 +40,7 @@ public interface CalculatedFieldState { Map getArguments(); - default boolean isValid(Map arguments) { - return getArguments().keySet().containsAll(arguments.keySet()); - } - - void initState(Map argumentValues); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index b9b98f9c5e..87429050de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -39,26 +39,23 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { - Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); - if (tsRecords.size() > argument.getLimit()) { - tsRecords.pollFirstEntry(); - } - tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + arguments.forEach((key, argumentEntry) -> { + if (argumentEntry instanceof TsRollingArgumentEntry) { + Argument argument = ctx.getArguments().get(key); + TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + if (tsRecords.size() > argument.getLimit()) { + tsRecords.pollFirstEntry(); } - }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); - ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); - Output output = ctx.getOutput(); - return Futures.transform(resultFuture, - result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), - MoreExecutors.directExecutor() - ); - } - return Futures.immediateFuture(null); + tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); + } + }); + Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); + Output output = ctx.getOutput(); + return Futures.transform(resultFuture, + result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), + MoreExecutors.directExecutor() + ); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 491419b40a..e16d310b3e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -37,27 +37,24 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - if (isValid(ctx.getArguments())) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); - - double expressionResult = expr.evaluate(); - - Output output = ctx.getOutput(); - return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); + String expression = ctx.getExpression(); + ThreadLocal customExpression = new ThreadLocal<>(); + var expr = customExpression.get(); + if (expr == null) { + expr = new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build(); + customExpression.set(expr); } - return Futures.immediateFuture(null); + Map variables = new HashMap<>(); + this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); + expr.setVariables(variables); + + double expressionResult = expr.evaluate(); + + Output output = ctx.getOutput(); + return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult))); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e0db8c50fb..e6cb24b970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -16,19 +16,18 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @Data +@NoArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - public SingleValueArgumentEntry() { - } - public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry) { this.ts = ((TsKvEntry) entry).getTs(); @@ -48,4 +47,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return this.ts != ((SingleValueArgumentEntry) entry).getTs(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1166da113d..104d0ae90c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -40,4 +40,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return tsRecords; } + @Override + public boolean hasUpdatedValue(ArgumentEntry entry) { + return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index ad64204e0f..b94b319e71 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -32,11 +32,7 @@ import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -44,7 +40,6 @@ import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -52,14 +47,11 @@ import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.stats.TbApiUsageReportClient; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import java.util.ArrayList; @@ -73,7 +65,6 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; /** * Created by ashvayka on 27.03.18. @@ -87,11 +78,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private ExecutorService tsCallBackExecutor; @Value("${sql.ts.value_no_xss_validation:false}") @@ -102,19 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldService calculatedFieldService, - CalculatedFieldExecutionService calculatedFieldExecutionService, - TbAssetProfileCache assetProfileCache, - TbDeviceProfileCache deviceProfileCache) { + CalculatedFieldExecutionService calculatedFieldExecutionService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldService = calculatedFieldService; this.calculatedFieldExecutionService = calculatedFieldExecutionService; - this.assetProfileCache = assetProfileCache; - this.deviceProfileCache = deviceProfileCache; } @PostConstruct @@ -201,7 +182,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); addEntityViewCallback(tenantId, entityId, ts); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } private void saveWithoutLatestAndNotifyInternal(TenantId tenantId, EntityId entityId, List ts, long ttl, FutureCallback callback) { @@ -210,55 +191,6 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); } - private void updateTelemetryInCalculatedFields(TenantId tenantId, EntityId entityId, List telemetry) { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); - if (!cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); - Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); - } - }); - } - } - } - - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - private void addEntityViewCallback(TenantId tenantId, EntityId entityId, List ts) { if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), @@ -335,7 +267,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -343,7 +275,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(tenantId, entityId, scope, attributes); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope.name(), attributes, notifyDevice)); - updateTelemetryInCalculatedFields(tenantId, entityId, attributes); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, attributes); } @Override @@ -357,7 +289,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = tsService.saveLatest(tenantId, entityId, ts); addVoidCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); - updateTelemetryInCalculatedFields(tenantId, entityId, ts); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, ts); } @Override From 481753b8f0e0b15f3174f65ee18fca33eb0a7568 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 17 Dec 2024 13:41:23 +0200 Subject: [PATCH 055/438] added restriction for number of cfs/arguments/ts rolling values --- .../service/cf/ctx/state/ArgumentEntry.java | 3 ++ .../ctx/state/BaseCalculatedFieldState.java | 9 ++--- .../ctx/state/SingleValueArgumentEntry.java | 8 ++++ .../cf/ctx/state/TsRollingArgumentEntry.java | 37 +++++++++++++++++-- .../cf/DefaultTbCalculatedFieldService.java | 18 +++++++++ 5 files changed, 67 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index 78222244c9..ba7e094f77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -52,4 +52,7 @@ public interface ArgumentEntry { collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); } + @JsonIgnore + ArgumentEntry copy(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ae6fc9033a..b73ac51798 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -42,25 +42,24 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingArgumentEntry != null) { if (existingArgumentEntry instanceof SingleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords()); + existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); stateUpdated.set(true); } } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); + existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); stateUpdated.set(true); } - } } } else { - arguments.put(key, argumentEntry); + arguments.put(key, argumentEntry.copy()); stateUpdated.set(true); } }); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index e6cb24b970..81a57580db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -23,6 +24,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @Data @NoArgsConstructor +@AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; @@ -51,4 +53,10 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return this.ts != ((SingleValueArgumentEntry) entry).getTs(); } + + @Override + public ArgumentEntry copy() { + return new SingleValueArgumentEntry(this.ts, this.value); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 104d0ae90c..49aae15ac1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,18 +16,26 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor -@AllArgsConstructor +@Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { - private TreeMap tsRecords; + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; + + private TreeMap tsRecords = new TreeMap<>(); + + public TsRollingArgumentEntry(TreeMap tsRecords) { + addAllTsRecords(tsRecords); + } @Override public ArgumentType getType() { @@ -44,4 +52,27 @@ public class TsRollingArgumentEntry implements ArgumentEntry { public boolean hasUpdatedValue(ArgumentEntry entry) { return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); } + + @Override + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + } + + public void addTsRecord(Long key, Object value) { + if (NumberUtils.isParsable(value.toString())) { + tsRecords.put(key, value); + if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { + tsRecords.pollFirstEntry(); + } + } else { + log.warn("Argument type 'TS_ROLLING' only supports numeric values."); + } + } + + public void addAllTsRecords(Map newRecords) { + for (Map.Entry entry : newRecords.entrySet()) { + addTsRecord(entry.getKey(), entry.getValue()); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 4d28ff55ac..2e6e975636 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -46,6 +46,9 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { + private static final int MAX_ARGUMENT_SIZE = 10; + private static final int MAX_CALCULATED_FIELD_NUMBER = 10; + private final CalculatedFieldService calculatedFieldService; @Override @@ -53,7 +56,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); + checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -105,6 +110,19 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } + private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { + if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { + throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); + } + } + + private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { + int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); + if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { + throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); + } + } + private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); From f929de42097c4929cae2d8970ea45dfced2b12b0 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 18 Dec 2024 13:12:39 +0200 Subject: [PATCH 056/438] onAddedPartitions() impl --- ...efaultCalculatedFieldExecutionService.java | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c1a86ac36f..7e5ea5b5ec 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -63,6 +64,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.asset.AssetService; @@ -173,13 +175,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { - // TODO: implementation for cluster mode - return Map.of(); + var result = new HashMap>>(); + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + Map> tpiCalculatedFieldMap = new HashMap<>(); + + for (CalculatedField cf : cfs) { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + cf.getId(), cf.getTenantId(), e.getMessage()); + continue; + } + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + } + } + + for (var entry : tpiCalculatedFieldMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { + log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); + var future = calculatedFieldExecutor.submit(() -> { + try { + for (CalculatedField cf : partition) { + if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + .forEach(entityId -> restoreState(cf, entityId)); + } else { + restoreState(cf, cf.getEntityId()); + } + } + } catch (Throwable t) { + log.error("Unexpected exception while restoring CalculatedField states", t); + throw t; + } + }); + result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); + } + } + return result; + } + + private void restoreState(CalculatedField cf, EntityId entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + + if (storedState != null) { + CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); + states.put(ctxId, restoredCtx); + log.info("Restored state for CalculatedField [{}]", cf.getId()); + } else { + log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + } } @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - // TODO: implementation for cluster mode + calculatedFields.remove(entityId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); } @Override @@ -213,8 +268,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { - initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); + getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); }); }); } From 5c5dc474cbc2ba899e9aa61106a211981e122dec Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 19 Dec 2024 17:12:59 +0200 Subject: [PATCH 057/438] implemented partitioning --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++--- .../queue/DefaultTbCoreConsumerService.java | 16 ++- common/proto/src/main/proto/queue.proto | 12 ++ 4 files changed, 125 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 302e4b2511..966b5d7277 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -29,7 +29,9 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); - void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); + + void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7e5ea5b5ec..f2a09d0faf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -98,6 +98,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -184,7 +185,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant [{}]. Reason: {}", + log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", cf.getId(), cf.getTenantId(), e.getMessage()); continue; } @@ -233,8 +234,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - calculatedFields.remove(entityId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(entityId.getId())); + cleanupEntity(entityId); + } + + private void cleanupEntity(CalculatedFieldId calculatedFieldId) { + calculatedFields.remove(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @Override @@ -245,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); + onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); @@ -364,7 +369,34 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + String state = proto.getState(); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + if (calculatedFieldEntityCtx != null) { + states.put(ctxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + } else { + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + } else { + log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); + } + } catch (Exception e) { + log.trace("Failed to process calculated field update state msg: [{}]", proto, e); + } + } + + @Override + public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); @@ -377,9 +409,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } }); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); @@ -398,12 +435,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.get(profileId).remove(entityId); - List statesToRemove = states.keySet().stream() - .filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); - rocksDBService.deleteAll(statesToRemove); + List calculatedFieldIds = Stream.concat( + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + ).toList(); + calculatedFieldIds.forEach(cfId -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + } + }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); @@ -414,11 +461,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + if (calculatedFieldState != null) { + msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + } + clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); @@ -428,8 +490,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } calculatedFields.remove(calculatedFieldId); calculatedFieldsCtx.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); @@ -606,7 +674,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId()); + CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -616,8 +685,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); if (allArgsPresent) { @@ -626,7 +701,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(tenantId, entityId, result); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6cf42cb893..a686184b8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -322,6 +322,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { @@ -708,6 +710,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 76da257b32..153394525d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -807,6 +807,17 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message CalculatedFieldStateMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; + string state = 8; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1541,6 +1552,7 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; + CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ From 2a41c8b45123e43fc34def1a954e098298c7339e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 20 Dec 2024 16:16:58 +0200 Subject: [PATCH 058/438] implemented logic to fetch telemetry if states were not fetched --- ...efaultCalculatedFieldExecutionService.java | 171 ++++++++++-------- .../ctx/state/SingleValueArgumentEntry.java | 10 + .../cf/ctx/state/TsRollingArgumentEntry.java | 13 +- 3 files changed, 118 insertions(+), 76 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f2a09d0faf..1128a75008 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -81,12 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -97,6 +98,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -200,11 +202,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas var future = calculatedFieldExecutor.submit(() -> { try { for (CalculatedField cf : partition) { - if (EntityType.ASSET_PROFILE.equals(cf.getEntityId().getEntityType()) || EntityType.DEVICE_PROFILE.equals(cf.getEntityId().getEntityType())) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cf.getEntityId()) + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { - restoreState(cf, cf.getEntityId()); + restoreState(cf, cfEntityId); } } } catch (Throwable t) { @@ -272,9 +275,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { + Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArguments, callback); + initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); } @@ -473,7 +479,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldState != null) { msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); } - clusterService.pushMsgToCore(tenantId, entityId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { @@ -556,35 +562,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } - private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && - !EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { - futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { - onComplete.accept(argumentValues); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to fetch common arguments", t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); } @@ -595,7 +572,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCtx.getArguments().forEach((key, argument) -> { if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument), + futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), result -> { argumentValues.put(key, result); return result; @@ -618,10 +595,25 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) + EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); @@ -654,11 +646,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); } private KvEntry createDefaultKvEntry(Argument argument) { @@ -676,48 +674,67 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Predicate> allArgsPresent = (args) -> + args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + TenantId tenantId = calculatedFieldCtx.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + } + + if (allArgsPresent.test(state.getArguments())) { + performCalculation(calculatedFieldCtx, state, entityId); + } + } + }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - if (state == null) { - state = createStateByType(calculatedFieldCtx.getCfType()); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; } - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldCtx.getCfId()); - if (tpi.isMyPartition()) { - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); - } + performUpdateState.accept(state); + } - boolean allArgsPresent = calculatedFieldCtx.getArguments().keySet().containsAll(state.getArguments().keySet()); - if (allArgsPresent) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(tenantId, entityId, result); - } - } + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); + Futures.addCallback(resultFuture, new FutureCallback<>() { + @Override + public void onSuccess(CalculatedFieldResult result) { + if (result != null) { + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + } + } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); } - } + }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) { + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, null); + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } @@ -725,8 +742,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { try { String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTE".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTE".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); @@ -749,4 +766,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private boolean isProfileEntity(EntityId entityId) { + return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 81a57580db..3f4fd5bdce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; @AllArgsConstructor public class SingleValueArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); + private long ts; private Object value; @@ -39,6 +41,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.value = entry.getValue(); } + /** + * Internal constructor to create immutable SingleValueArgumentEntry.EMPTY + * */ + private SingleValueArgumentEntry(int ignored) { + this.ts = System.currentTimeMillis(); + this.value = null; + } + @Override public ArgumentType getType() { return ArgumentType.SINGLE_VALUE; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 49aae15ac1..4ffa391550 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -29,6 +29,8 @@ import java.util.TreeMap; @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { + public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0); + private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; private TreeMap tsRecords = new TreeMap<>(); @@ -37,6 +39,13 @@ public class TsRollingArgumentEntry implements ArgumentEntry { addAllTsRecords(tsRecords); } + /** + * Internal constructor to create immutable TsRollingArgumentEntry.EMPTY + */ + private TsRollingArgumentEntry(int ignored) { + this.tsRecords = new TreeMap<>(); + } + @Override public ArgumentType getType() { return ArgumentType.TS_ROLLING; @@ -50,7 +59,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @Override public boolean hasUpdatedValue(ArgumentEntry entry) { - return !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()); + return entry instanceof SingleValueArgumentEntry ? + !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : + !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); } @Override From 2d65a6c457182e2f7412f083b1be7069c47d7f2d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 12:20:58 +0200 Subject: [PATCH 059/438] removed wildcard imports usage --- .../server/controller/BaseController.java | 62 ++++++++++++++++++- .../entitiy/EntityStateSourcingListener.java | 9 ++- .../DefaultTelemetrySubscriptionService.java | 17 +++-- .../rule/engine/util/TenantIdLoader.java | 33 +++++++++- .../rule/engine/util/TenantIdLoaderTest.java | 27 +++++++- 5 files changed, 136 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 2d990d708d..9a980d009d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -42,7 +42,27 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.HomeDashboardInfo; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmInfo; @@ -57,7 +77,37 @@ import org.thingsboard.server.common.data.edge.EdgeInfo; import org.thingsboard.server.common.data.exception.EntityVersionMismatchException; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmCommentId; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; @@ -140,7 +190,13 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 83ce48c24e..ab8246ccf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -23,7 +23,14 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.cf.CalculatedField; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf66a9f868..2ddfd22db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -29,7 +29,11 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.rule.engine.api.*; +import org.thingsboard.rule.engine.api.AttributesDeleteRequest; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; +import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -49,7 +53,13 @@ import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -231,8 +241,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) { - } + public void onSuccess(@Nullable Void tmp) {} @Override public void onFailure(Throwable t) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java index 4311d912e4..445205e88e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/TenantIdLoader.java @@ -19,7 +19,38 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.DomainId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.MobileAppBundleId; +import org.thingsboard.server.common.data.id.MobileAppId; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.NotificationRuleId; +import org.thingsboard.server.common.data.id.NotificationTargetId; +import org.thingsboard.server.common.data.id.NotificationTemplateId; +import org.thingsboard.server.common.data.id.OAuth2ClientId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.QueueId; +import org.thingsboard.server.common.data.id.QueueStatsId; +import org.thingsboard.server.common.data.id.RpcId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.rule.RuleNode; import java.util.UUID; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index a879c0e3a2..921b9e26d7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -23,8 +23,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.rule.engine.api.*; -import org.thingsboard.server.common.data.*; +import org.thingsboard.rule.engine.api.RuleEngineAlarmService; +import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; +import org.thingsboard.rule.engine.api.RuleEngineAssetProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; +import org.thingsboard.rule.engine.api.RuleEngineRpcService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; @@ -32,7 +47,13 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.domain.Domain; import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.NotificationId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.mobile.bundle.MobileAppBundle; import org.thingsboard.server.common.data.notification.NotificationRequest; From 1482d1c4bb9983f18ab9d6abb5405f1a68496ec9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 23 Dec 2024 17:00:10 +0200 Subject: [PATCH 060/438] added logic to avoid looping --- .../cf/CalculatedFieldExecutionService.java | 3 +- ...efaultCalculatedFieldExecutionService.java | 57 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 4 +- .../thingsboard/server/common/msg/TbMsg.java | 30 ++++++++-- common/message/src/main/proto/tbmsg.proto | 7 +++ .../engine/api/AttributesSaveRequest.java | 10 +++- .../engine/api/TimeseriesSaveRequest.java | 10 +++- .../engine/telemetry/TbMsgTimeseriesNode.java | 1 + 8 files changed, 76 insertions(+), 46 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 966b5d7277..ee9b6d115c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; @@ -27,7 +28,7 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry); + void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 1128a75008..e5c119bba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,11 +18,7 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.*; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -41,25 +37,8 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.id.*; +import org.thingsboard.server.common.data.kv.*; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -301,7 +280,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List telemetry) { + public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { EntityType entityType = entityId.getEntityType(); if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { @@ -326,7 +305,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, updatedTelemetry); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); } }); } @@ -335,7 +314,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); @@ -347,12 +326,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); + getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues); + default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @@ -583,7 +562,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); callback.onSuccess(); } @@ -671,7 +650,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); @@ -693,7 +672,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId); + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } }; @@ -714,13 +693,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performUpdateState.accept(state); } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); } } @@ -739,13 +718,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } - private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload)); + if (calculatedFieldIds.contains(calculatedFieldId)) { + throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); + } + calculatedFieldIds.add(calculatedFieldId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 2ddfd22db3..e35d3cead6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -152,7 +152,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); return saveFuture; } @@ -168,7 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); } @Override diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 64e05770fe..993fa24ff3 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -64,6 +66,8 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; + private final List calculatedFieldIds; + @Getter(value = AccessLevel.NONE) @JsonIgnore //This field is not serialized because we use queues and there is no need to do it @@ -112,7 +116,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -139,6 +143,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; + this.calculatedFieldIds = calculatedFieldIds; this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -200,6 +205,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; + List calculatedFieldIds = new ArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -214,6 +220,14 @@ public final class TbMsg implements Serializable { partition = proto.getPartition(); } + for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(calculatedFieldId); + } + TbMsgProcessingCtx ctx; if (proto.hasCtx()) { ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); @@ -224,7 +238,7 @@ public final class TbMsg implements Serializable { TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, - metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } catch (InvalidProtocolBufferException e) { throw new IllegalStateException("Could not parse protobuf for TbMsg", e); } @@ -343,10 +357,12 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; + protected List calculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; - TbMsgBuilder() {} + TbMsgBuilder() { + } TbMsgBuilder(TbMsg tbMsg) { this.queueName = tbMsg.queueName; @@ -363,6 +379,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; + this.calculatedFieldIds = tbMsg.calculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -454,6 +471,11 @@ public final class TbMsg implements Serializable { return this; } + public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { this.ctx = ctx; return this; @@ -465,7 +487,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); } public String toString() { diff --git a/common/message/src/main/proto/tbmsg.proto b/common/message/src/main/proto/tbmsg.proto index fc9265aa14..36ab09f8d4 100644 --- a/common/message/src/main/proto/tbmsg.proto +++ b/common/message/src/main/proto/tbmsg.proto @@ -70,4 +70,11 @@ message TbMsgProto { int64 correlationIdMSB = 20; int64 correlationIdLSB = 21; int32 partition = 22; + + repeated CalculatedFieldIdProto calculatedFields = 23; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 22fa8de6de..9747a6033e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -22,6 +22,7 @@ import lombok.AllArgsConstructor; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -53,6 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; + private List calculatedFieldIds; private FutureCallback callback; Builder() {} @@ -100,6 +103,11 @@ public class AttributesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -120,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 2b5881212d..12afa2d939 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Getter; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -40,6 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; + private final List calculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; + private List calculatedFieldIds; Builder() {} @@ -103,6 +106,11 @@ public class TimeseriesSaveRequest { return this; } + public Builder calculatedFieldIds(List calculatedFieldIds) { + this.calculatedFieldIds = calculatedFieldIds; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -123,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 27f45feb47..386e56320a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,6 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From a9f39e4917d0da31bdc89a45a8c5df344eec9d1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 12:50:27 +0200 Subject: [PATCH 061/438] added cache class --- .../service/cf/CalculatedFieldCache.java | 43 +++ .../cf/DefaultCalculatedFieldCache.java | 212 +++++++++++++ ...efaultCalculatedFieldExecutionService.java | 279 ++++++++++-------- .../thingsboard/server/common/msg/TbMsg.java | 15 +- common/proto/src/main/proto/queue.proto | 29 +- .../engine/telemetry/TbMsgAttributesNode.java | 1 + 6 files changed, 454 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java new file mode 100644 index 0000000000..f953e57cc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; +import java.util.Set; + +public interface CalculatedFieldCache { + + CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + + void evict(CalculatedFieldId calculatedFieldId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java new file mode 100644 index 0000000000..1fc2a90e07 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -0,0 +1,212 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import jakarta.annotation.PostConstruct; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + + private final Lock calculatedFieldFetchLock = new ReentrantLock(); + + private final CalculatedFieldService calculatedFieldService; + private final AssetService assetService; + private final DeviceService deviceService; + + private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + + @PostConstruct + public void init() { + // to discuss: fetch on start or fetch on demand + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + } + + @Override + public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedFieldFetchLock.lock(); + try { + calculatedField = calculatedFields.get(calculatedFieldId); + if (calculatedField == null) { + calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + if (calculatedField != null) { + calculatedFields.put(calculatedFieldId, calculatedField); + log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); + return calculatedField; + } + + @Override + public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + List cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = calculatedFieldLinks.get(calculatedFieldId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); + if (cfLinks != null) { + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); + return cfLinks; + } + + @Override + public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { + List cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + calculatedFieldFetchLock.lock(); + try { + cfLinks = entityIdCalculatedFieldLinks.get(entityId); + if (cfLinks == null) { + cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); + if (cfLinks != null) { + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); + return cfLinks; + } + + @Override + public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + calculatedFieldFetchLock.lock(); + try { + ctx = calculatedFieldsCtx.get(calculatedFieldId); + if (ctx == null) { + CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + if (calculatedField != null) { + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + calculatedFieldsCtx.put(calculatedFieldId, ctx); + log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx); + return ctx; + } + + @Override + public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + calculatedFieldFetchLock.lock(); + try { + entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> + throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + @Override + public void evict(CalculatedFieldId calculatedFieldId) { + CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); + calculatedFieldsCtx.remove(calculatedFieldId); + log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId))); + log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e5c119bba6..ff78d2925a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -18,7 +18,11 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; -import com.google.common.util.concurrent.*; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; @@ -37,8 +41,23 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.*; -import org.thingsboard.server.common.data.kv.*; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; @@ -46,10 +65,8 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -67,8 +84,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,7 +94,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -90,10 +106,9 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final RocksDBService rocksDBService; @@ -103,13 +118,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); - private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); private final ConcurrentMap states = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; @Value("${calculatedField.initFetchPackSize:50000}") @@ -123,20 +133,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(this::fetchCalculatedFields); - } - - private void fetchCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); - rocksDBService.getAll().forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class))); - states.keySet().removeIf(ctxId -> calculatedFields.keySet().stream().noneMatch(id -> ctxId.cfId().equals(id.getId()))); } @PreDestroy public void stop() { + super.stop(); if (calculatedFieldExecutor != null) { calculatedFieldExecutor.shutdownNow(); } @@ -183,7 +184,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas for (CalculatedField cf : partition) { EntityId cfEntityId = cf.getEntityId(); if (isProfileEntity(cfEntityId)) { - getOrFetchFromDBProfileEntities(cf.getTenantId(), cfEntityId) + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) .forEach(entityId -> restoreState(cf, entityId)); } else { restoreState(cf, cfEntityId); @@ -206,7 +207,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); - calculatedFieldsCtx.putIfAbsent(cf.getId(), new CalculatedFieldCtx(cf, tbelInvokeService)); states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", cf.getId()); } else { @@ -220,7 +220,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - calculatedFields.remove(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); } @@ -235,7 +234,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -245,8 +244,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); - calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -258,7 +256,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(targetEntityId -> { + calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); }); }); @@ -290,8 +288,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = new ArrayList<>(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> cfLinks.addAll(calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, id))); + List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); + Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -316,8 +314,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -326,7 +324,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); } else { updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); } @@ -353,28 +351,61 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entry.getKey(); } + private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { + try { + String type = objectProto.getType(); + String value = objectProto.getValue(); + return switch (type) { + case "java.lang.String" -> value; + case "java.lang.Integer" -> Integer.parseInt(value); + case "java.lang.Long" -> Long.parseLong(value); + case "java.lang.Double" -> Double.parseDouble(value); + case "java.lang.Boolean" -> Boolean.parseBoolean(value); + default -> throw new IllegalArgumentException("Unsupported object type: " + type); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); + return null; + } + } + @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - String state = proto.getState(); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = state.isEmpty() ? JacksonUtil.fromString(state, CalculatedFieldEntityCtx.class) : null; + List calculatedFieldIds = new ArrayList<>(); + for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { + CalculatedFieldId cfId = new CalculatedFieldId(new UUID( + cfIdProto.getCalculatedFieldIdMSB(), + cfIdProto.getCalculatedFieldIdLSB() + )); + calculatedFieldIds.add(cfId); + } - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); - if (calculatedFieldEntityCtx != null) { - states.put(ctxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), state); + Map argumentsMap = new HashMap<>(); + proto.getArgumentsMap().forEach((key, entryProto) -> { + ArgumentEntry argumentEntry; + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { + Object value = deserializeObjectProto(objectProto); + tsRollingArgumentEntry.getTsRecords().put(ts, value); + }); + argumentEntry = tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); + Object value = deserializeObjectProto(singleRecordProto.getValue()); + argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); } else { - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } - } else { - log.debug("[{}] Calculated Field belongs to external partition {}", calculatedFieldId, tpi.getFullTopicName()); - } + argumentsMap.put(key, argumentEntry); + }); + + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -389,8 +420,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(oldProfileId).remove(entityId); - profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> { @@ -400,7 +431,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); @@ -419,12 +450,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.get(profileId).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); List calculatedFieldIds = Stream.concat( - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, profileId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); calculatedFieldIds.forEach(cfId -> { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); @@ -433,12 +462,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, null); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); } }); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -446,7 +475,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, CalculatedFieldState calculatedFieldState) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) @@ -455,20 +484,45 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .setEntityType(entityId.getEntityType().name()) .setEntityIdMSB(entityId.getId().getMostSignificantBits()) .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (calculatedFieldState != null) { - msgBuilder.setState(JacksonUtil.writeValueAsString(calculatedFieldState)); + + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> { + TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { + TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() + .setType(value.getClass().getName()) + .setValue(value.toString()); + tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); + }); + + argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(TransportProtos.ObjectProto.newBuilder() + .setType(singleValueArgumentEntry.getValue().getClass().getName()) + .setValue(singleValueArgumentEntry.getValue().toString()) + .build()); + argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); + } + + msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); + }); } + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); } else { - calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); - calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService)); callback.onSuccess(); shouldReinit = false; } @@ -483,8 +537,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFields.remove(calculatedFieldId); - calculatedFieldsCtx.remove(calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) @@ -497,28 +550,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId)); - } - - private Set getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) { - return switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { if (oldCalculatedField == null) { return true; @@ -537,7 +568,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -562,7 +593,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, Collections.emptyList()); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -651,46 +682,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Predicate> allArgsPresent = (args) -> - args.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !args.containsValue(SingleValueArgumentEntry.EMPTY) && !args.containsValue(TsRollingArgumentEntry.EMPTY); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - TenantId tenantId = calculatedFieldCtx.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, state); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } + }; - if (allArgsPresent.test(state.getArguments())) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); - } - } - }; + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + if (!allKeysPresent) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - return; + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + return; + } + performUpdateState.accept(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + } else { + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } - performUpdateState.accept(state); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { @@ -724,6 +756,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); + if (calculatedFieldIds == null) { + calculatedFieldIds = new ArrayList<>(); + } if (calculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 993fa24ff3..0805175f77 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -191,6 +192,16 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } + if (msg.getCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .build(); + builder.addCalculatedFields(calculatedFieldIdProto); + } + } + builder.setCtx(msg.ctx.toProto()); return builder.build().toByteArray(); } @@ -495,8 +506,8 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + - ", callback=" + this.callback + ")"; + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 3df19e313e..f6f7afd5d8 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,7 +817,34 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - string state = 8; + repeated CalculatedFieldIdProto calculatedFields = 8; + map arguments = 9; +} + +message CalculatedFieldIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; +} + +message ArgumentEntryProto { + oneof entry_type { + TsRollingProto tsRecords = 1; + SingleValueProto singleValue = 2; + } +} + +message TsRollingProto { + map tsRecords = 1; +} + +message SingleValueProto { + int64 ts = 1; + ObjectProto value = 2; +} + +message ObjectProto { + string type = 1; + string value = 2; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index e83a125265..20d0dda42f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,6 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) + .calculatedFieldIds(msg.getCalculatedFieldIds()) .callback(callback) .build()); } From 2dfbe2240d4bd22904e3d5edd6a7edeb15b0d1a5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 24 Dec 2024 16:25:01 +0200 Subject: [PATCH 062/438] added implementation to handle profile events when not my partition --- ...efaultCalculatedFieldExecutionService.java | 510 +++++++++--------- .../server/common/util/ProtoUtils.java | 40 ++ common/proto/src/main/proto/queue.proto | 14 +- 3 files changed, 306 insertions(+), 258 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index ff78d2925a..6e7536825a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -84,7 +84,6 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -98,6 +97,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; @TbCoreComponent @Service @@ -229,6 +230,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (!tpi.isMyPartition()) { + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); + log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); + callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); + } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -277,6 +284,54 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + boolean shouldReinit = true; + if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { + onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + } else { + callback.onSuccess(); + shouldReinit = false; + } + return shouldReinit; + } + + private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + try { + cleanupEntity(calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + Set calculatedFieldIds = partitionedEntities.get(tpi); + if (calculatedFieldIds != null) { + calculatedFieldIds.remove(calculatedFieldId); + } + calculatedFieldCache.evict(calculatedFieldId); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + List statesToRemove = states.keySet().stream() + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .map(JacksonUtil::writeValueAsString) + .toList(); + rocksDBService.deleteAll(statesToRemove); + } catch (Exception e) { + log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); + callback.onFailure(e); + } + } + + private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (oldCalculatedField == null) { + return true; + } + boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); + boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); + CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); + CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); + boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); + boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); + boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + + return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + } + @Override public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { try { @@ -288,8 +343,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else if (EntityType.DEVICE.equals(entityType)) { profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); } - List cfLinks = calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId); - Optional.ofNullable(profileId).ifPresent(id -> calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + Optional.ofNullable(profileId).ifPresent(id -> { + cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); + }); cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); Map attributes = link.getConfiguration().getAttributes(); @@ -312,6 +369,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { + if (entry instanceof AttributeKvEntry) { + return attributes.entrySet().stream() + .filter(attr -> attr.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } else if (entry instanceof TsKvEntry) { + return timeSeries.entrySet().stream() + .filter(ts -> ts.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); + } + return entry.getKey(); + } + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -334,75 +408,23 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); - } - - private Object deserializeObjectProto(TransportProtos.ObjectProto objectProto) { - try { - String type = objectProto.getType(); - String value = objectProto.getValue(); - return switch (type) { - case "java.lang.String" -> value; - case "java.lang.Integer" -> Integer.parseInt(value); - case "java.lang.Long" -> Long.parseLong(value); - case "java.lang.Double" -> Double.parseDouble(value); - case "java.lang.Boolean" -> Boolean.parseBoolean(value); - default -> throw new IllegalArgumentException("Unsupported object type: " + type); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", objectProto, e); - return null; - } - } - @Override public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - List calculatedFieldIds = new ArrayList<>(); - for (TransportProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) { - CalculatedFieldId cfId = new CalculatedFieldId(new UUID( - cfIdProto.getCalculatedFieldIdMSB(), - cfIdProto.getCalculatedFieldIdLSB() - )); - calculatedFieldIds.add(cfId); + + if (proto.getClear()) { + clearState(tenantId, calculatedFieldId, entityId); + return; } - Map argumentsMap = new HashMap<>(); - proto.getArgumentsMap().forEach((key, entryProto) -> { - ArgumentEntry argumentEntry; - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> { - Object value = deserializeObjectProto(objectProto); - tsRollingArgumentEntry.getTsRecords().put(ts, value); - }); - argumentEntry = tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleRecordProto = entryProto.getSingleValue(); - Object value = deserializeObjectProto(singleRecordProto.getValue()); - argumentEntry = new SingleValueArgumentEntry(singleRecordProto.getTs(), value); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - argumentsMap.put(key, argumentEntry); - }); + List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList(); + Map argumentsMap = proto.getArgumentsMap().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); @@ -424,16 +446,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + .forEach(cfId -> clearState(tenantId, cfId, entityId)); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); } catch (Exception e) { @@ -455,16 +468,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); - calculatedFieldIds.forEach(cfId -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); - if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, Collections.emptyList(), null); - } - }); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); @@ -475,94 +479,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> { - TransportProtos.ArgumentEntryProto.Builder argumentEntryProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> { - TransportProtos.ObjectProto.Builder objectProtoBuilder = TransportProtos.ObjectProto.newBuilder() - .setType(value.getClass().getName()) - .setValue(value.toString()); - tsRollingProtoBuilder.putTsRecords(ts, objectProtoBuilder.build()); - }); - - argumentEntryProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - TransportProtos.SingleValueProto.Builder singleRecordProtoBuilder = TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(TransportProtos.ObjectProto.newBuilder() - .setType(singleValueArgumentEntry.getValue().getClass().getName()) - .setValue(singleValueArgumentEntry.getValue().toString()) - .build()); - argumentEntryProtoBuilder.setSingleValue(singleRecordProtoBuilder.build()); - } - - msgBuilder.putArguments(key, argumentEntryProtoBuilder.build()); - }); - } - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + if (tpi.isMyPartition()) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + states.remove(ctxId); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { - callback.onSuccess(); - shouldReinit = false; + sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } - return shouldReinit; - } - - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } - calculatedFieldCache.evict(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); - - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; } private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { @@ -605,82 +531,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( - Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor) - ); - case "TS_LATEST" -> transformSingleValueArgument( - Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), - calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); - }; - } - - private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { - long currentTime = System.currentTimeMillis(); - long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); - long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); - ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); - - return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); - } - - private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { - return Futures.transform(kvEntryFuture, kvEntry -> { - if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { - return ArgumentEntry.createSingleValueArgument(kvEntry.get()); - } else { - return SingleValueArgumentEntry.EMPTY; - } - }, calculatedFieldCallbackExecutor); - } - - private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); - String defaultValue = argument.getDefaultValue(); - if (NumberUtils.isParsable(defaultValue)) { - return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); - } - if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { - return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); - } - return new StringDataEntry(key, defaultValue); - } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); @@ -742,14 +592,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); - } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { String type = calculatedFieldResult.getType(); @@ -770,6 +612,166 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { + Map argumentValues = new HashMap<>(); + List> futures = new ArrayList<>(); + necessaryArguments.forEach((key, argument) -> { + futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), + result -> { + argumentValues.put(key, result); + return result; + }, calculatedFieldCallbackExecutor)); + }); + return Futures.transform(Futures.allAsList(futures), results -> { + onComplete.accept(argumentValues); + return null; + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { + EntityId argumentEntityId = argument.getEntityId(); + EntityId entityId = isProfileEntity(argumentEntityId) + ? targetEntityId + : argumentEntityId; + return fetchKvEntry(tenantId, entityId, argument); + } + + private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { + return switch (argument.getType()) { + case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); + case "ATTRIBUTE" -> transformSingleValueArgument( + Futures.transform( + attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor) + ); + case "TS_LATEST" -> transformSingleValueArgument( + Futures.transform( + timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + calculatedFieldCallbackExecutor)); + default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); + }; + } + + private ListenableFuture transformSingleValueArgument(ListenableFuture> kvEntryFuture) { + return Futures.transform(kvEntryFuture, kvEntry -> { + if (kvEntry.isPresent() && kvEntry.get().getValue() != null) { + return ArgumentEntry.createSingleValueArgument(kvEntry.get()); + } else { + return SingleValueArgumentEntry.EMPTY; + } + }, calculatedFieldCallbackExecutor); + } + + private ListenableFuture fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { + long currentTime = System.currentTimeMillis(); + long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); + long startTs = currentTime - timeWindow; + int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); + + return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); + } + + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); + if (argumentValues != null) { + argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); + } + if (calculatedFieldIds != null) { + calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build() + )); + } + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); + } + + private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { + TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) + .setClear(true) + .build(); + + clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); + } + + private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( + TenantId tenantId, + CalculatedFieldId calculatedFieldId, + EntityId entityId + ) { + return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + } + + private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { + TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); + + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { + TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); + tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> + tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) + ); + argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); + } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + argumentProtoBuilder.setSingleValue( + TransportProtos.SingleValueProto.newBuilder() + .setTs(singleValueArgumentEntry.getTs()) + .setValue(toObjectProto(singleValueArgumentEntry.getValue())) + .build() + ); + } + + return argumentProtoBuilder.build(); + } + + private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { + if (entryProto.hasTsRecords()) { + TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); + entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> + tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) + ); + return tsRollingArgumentEntry; + } else if (entryProto.hasSingleValue()) { + TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + } else { + throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); + } + } + + private KvEntry createDefaultKvEntry(Argument argument) { + String key = argument.getKey(); + String defaultValue = argument.getDefaultValue(); + if (NumberUtils.isParsable(defaultValue)) { + return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); + } + if ("true".equalsIgnoreCase(defaultValue) || "false".equalsIgnoreCase(defaultValue)) { + return new BooleanDataEntry(key, Boolean.parseBoolean(defaultValue)); + } + return new StringDataEntry(key, defaultValue); + } + + private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + } + private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 264b23f118..ec17914fd8 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1183,6 +1183,46 @@ public class ProtoUtils { return builder.build(); } + public static TransportProtos.ObjectProto toObjectProto(Object value) { + if (value == null) { + throw new IllegalArgumentException("Cannot convert null to ObjectProto"); + } + + TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); + + if (value instanceof String) { + builder.setStringValue((String) value); + } else if (value instanceof Integer) { + builder.setIntValue((Integer) value); + } else if (value instanceof Long) { + builder.setLongValue((Long) value); + } else if (value instanceof Double) { + builder.setDoubleValue((Double) value); + } else if (value instanceof Boolean) { + builder.setBoolValue((Boolean) value); + } else { + throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); + } + + return builder.build(); + } + + public static Object fromObjectProto(TransportProtos.ObjectProto proto) { + try { + return switch (proto.getValueCase()) { + case STRINGVALUE -> proto.getStringValue(); + case INTVALUE -> proto.getIntValue(); + case LONGVALUE -> proto.getLongValue(); + case DOUBLEVALUE -> proto.getDoubleValue(); + case BOOLVALUE -> proto.getBoolValue(); + case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); + }; + } catch (Exception e) { + log.error("Failed to deserialize ObjectProto: [{}]", proto, e); + return null; + } + } + private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index f6f7afd5d8..581f9eb9f2 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -817,8 +817,9 @@ message CalculatedFieldStateMsgProto { string entityType = 5; int64 entityIdMSB = 6; int64 entityIdLSB = 7; - repeated CalculatedFieldIdProto calculatedFields = 8; - map arguments = 9; + bool clear = 8; + repeated CalculatedFieldIdProto calculatedFields = 9; + map arguments = 10; } message CalculatedFieldIdProto { @@ -843,8 +844,13 @@ message SingleValueProto { } message ObjectProto { - string type = 1; - string value = 2; + oneof value { + string stringValue = 1; + int32 intValue = 2; + int64 longValue = 3; + double doubleValue = 4; + bool boolValue = 5; + } } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From befd4cc9c69749dc510c5ff6ae2ff67713264d06 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 10:56:20 +0200 Subject: [PATCH 063/438] added check for update values for ts rolling when rocksdb fails --- .../service/cf/DefaultCalculatedFieldCache.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 13 +++++++------ .../cf/ctx/state/BaseCalculatedFieldState.java | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 1fc2a90e07..dd2ab3857c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -100,11 +100,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); if (cfLinks != null) { calculatedFieldLinks.put(calculatedFieldId, cfLinks); @@ -122,11 +122,11 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { + if (cfLinks == null || cfLinks.isEmpty()) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); if (cfLinks != null) { entityIdCalculatedFieldLinks.put(entityId, cfLinks); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 6e7536825a..808c1ea8d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -556,20 +556,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - if (!allKeysPresent) { + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + + if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey())) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); - return; + } else { + performUpdateState.accept(state); } - performUpdateState.accept(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index b73ac51798..462fa19b17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -24,6 +24,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected Map arguments; public BaseCalculatedFieldState() { + arguments = new HashMap<>(); } @Override From e787b805dae819f083876168e0a960bd5c801ae1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 26 Dec 2024 17:05:27 +0200 Subject: [PATCH 064/438] added locking and logic to avoid iteration order issue in map --- ...efaultCalculatedFieldExecutionService.java | 68 +++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 5 +- .../ctx/state/BaseCalculatedFieldState.java | 49 ++++++------- .../cf/ctx/state/CalculatedFieldCtx.java | 8 ++- .../ctx/state/ScriptCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 ++-- .../cf/ctx/state/TsRollingArgumentEntry.java | 8 ++- common/proto/src/main/proto/queue.proto | 1 + 8 files changed, 87 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 808c1ea8d8..af4b3e95e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -119,6 +120,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; + private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); + private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; @@ -536,40 +539,47 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { - calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); + lock.lock(); + + try { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentValues)) { + calculatedFieldEntityCtx.setState(state); + states.put(entityCtxId, calculatedFieldEntityCtx); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + } } - } - }; + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { + if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } + } finally { + lock.unlock(); } } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); @@ -747,7 +757,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return tsRollingArgumentEntry; } else if (entryProto.hasSingleValue()) { TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue())); + return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); } else { throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index ba7e094f77..dd56405352 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -import java.util.TreeMap; -import java.util.stream.Collectors; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -48,8 +46,7 @@ public interface ArgumentEntry { } static ArgumentEntry createTsRollingArgument(List kvEntries) { - return new TsRollingArgumentEntry(kvEntries.stream(). - collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new))); + return new TsRollingArgumentEntry(kvEntries); } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 462fa19b17..be7319b930 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @@ -37,34 +36,30 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (arguments == null) { arguments = new HashMap<>(); } - AtomicBoolean stateUpdated = new AtomicBoolean(false); - argumentValues.forEach((key, argumentEntry) -> { - ArgumentEntry existingArgumentEntry = arguments.get(key); - if (existingArgumentEntry != null) { - if (existingArgumentEntry instanceof SingleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); - } - } else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addAllTsRecords(tsRollingArgumentEntry.getTsRecords()); - stateUpdated.set(true); - } - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - if (existingArgumentEntry.hasUpdatedValue(argumentEntry)) { - existingTsRollingArgumentEntry.addTsRecord(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); - stateUpdated.set(true); - } - } + + boolean stateUpdated = false; + + for (Map.Entry entry : argumentValues.entrySet()) { + String key = entry.getKey(); + ArgumentEntry newEntry = entry.getValue(); + ArgumentEntry existingEntry = arguments.get(key); + + if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { + if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { + existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); + } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { + existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry + && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + arguments.put(key, newEntry.copy()); + } else { + arguments.put(key, newEntry.copy()); } - } else { - arguments.put(key, argumentEntry.copy()); - stateUpdated.set(true); + stateUpdated = true; } - }); - return stateUpdated.get(); + } + + return stateUpdated; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 2cd5c68144..d54a3220ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.ArrayList; +import java.util.List; import java.util.Map; @Data @@ -36,6 +38,7 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final List argKeys; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -48,10 +51,11 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.argKeys = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { + if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); } } @@ -65,7 +69,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - arguments.keySet().toArray(new String[0]) + argKeys.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 87429050de..de7c514786 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); + Object[] args = ctx.getArgKeys().stream() + .map(key -> arguments.get(key).getValue()) + .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 3f4fd5bdce..20b531f562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,11 +32,15 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; + private long version; + public SingleValueArgumentEntry(KvEntry entry) { - if (entry instanceof TsKvEntry) { - this.ts = ((TsKvEntry) entry).getTs(); - } else if (entry instanceof AttributeKvEntry) { - this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); + if (entry instanceof TsKvEntry tsKvEntry) { + this.ts = tsKvEntry.getTs(); + this.version = tsKvEntry.getVersion(); + } else if (entry instanceof AttributeKvEntry attributeKvEntry) { + this.ts = attributeKvEntry.getLastUpdateTs(); + this.version = attributeKvEntry.getVersion(); } this.value = entry.getValue(); } @@ -66,7 +70,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value); + return new SingleValueArgumentEntry(this.ts, this.value, this.version); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 4ffa391550..64de2f8c8a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -16,16 +16,20 @@ package org.thingsboard.server.service.cf.ctx.state; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import java.util.List; import java.util.Map; import java.util.TreeMap; @Data @NoArgsConstructor +@AllArgsConstructor @Slf4j public class TsRollingArgumentEntry implements ArgumentEntry { @@ -35,8 +39,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); - public TsRollingArgumentEntry(TreeMap tsRecords) { - addAllTsRecords(tsRecords); + public TsRollingArgumentEntry(List kvEntries) { + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); } /** diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 581f9eb9f2..1f66621a5b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,6 +841,7 @@ message TsRollingProto { message SingleValueProto { int64 ts = 1; ObjectProto value = 2; + int64 version = 3; } message ObjectProto { From 5f088a751e2fc18435fecfeee603c592ef412714 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 27 Dec 2024 18:20:20 +0200 Subject: [PATCH 065/438] refactored code --- .../cf/CalculatedFieldExecutionService.java | 9 +- .../service/cf/CalculatedFieldResult.java | 5 +- ...efaultCalculatedFieldExecutionService.java | 122 ++++++++++-------- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- ...gumentType.java => ArgumentEntryType.java} | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 8 +- .../ctx/state/SingleValueArgumentEntry.java | 6 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 4 +- ...CalculatedFieldAttributeUpdateRequest.java | 38 ++++++ ...CalculatedFieldTelemetryUpdateRequest.java | 38 ++++++ ...alculatedFieldTimeSeriesUpdateRequest.java | 42 ++++++ .../DefaultTelemetrySubscriptionService.java | 6 +- .../CalculatedFieldControllerTest.java | 6 +- .../cf/CalculatedFieldLinkConfiguration.java | 4 +- .../data/cf/configuration/Argument.java | 2 +- .../data/cf/configuration/ArgumentType.java | 22 ++++ .../BaseCalculatedFieldConfiguration.java | 45 ++++--- .../common/data/cf/configuration/Output.java | 2 +- .../data/cf/configuration/OutputType.java | 22 ++++ .../server/dao/service/AssetServiceTest.java | 6 +- .../service/CalculatedFieldServiceTest.java | 6 +- .../dao/service/CustomerServiceTest.java | 6 +- .../server/dao/service/DeviceServiceTest.java | 6 +- 23 files changed, 302 insertions(+), 107 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{ArgumentType.java => ArgumentEntryType.java} (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index ee9b6d115c..e4b0a7ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; - -import java.util.List; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); - void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry); + void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index e8ea318bf6..52e0a0151b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -17,17 +17,18 @@ package org.thingsboard.server.service.cf; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import java.util.Map; @Data public final class CalculatedFieldResult { - private final String type; + private final OutputType type; private final AttributeScope scope; private final Map resultMap; - public CalculatedFieldResult(String type, AttributeScope scope, Map resultMap) { + public CalculatedFieldResult(OutputType type, AttributeScope scope, Map resultMap) { this.type = type; this.scope = scope; this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index af4b3e95e5..27db9ae290 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,12 +35,15 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,7 +51,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -79,11 +81,13 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,7 +96,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -120,12 +123,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap entityLocks = new ConcurrentHashMap<>(); - private final ConcurrentMap states = new ConcurrentHashMap<>(); private static final int MAX_LAST_RECORDS_VALUE = 1024; + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @@ -336,28 +341,29 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, List calculatedFieldIds, List telemetry) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - EntityType entityType = entityId.getEntityType(); - if (EntityType.DEVICE.equals(entityType) || EntityType.ASSET.equals(entityType) || EntityType.CUSTOMER.equals(entityType) || EntityType.TENANT.equals(entityType)) { - EntityId profileId = null; - if (EntityType.ASSET.equals(entityType)) { - profileId = assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - } else if (EntityType.DEVICE.equals(entityType)) { - profileId = deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - } - List cfLinks = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); - Optional.ofNullable(profileId).ifPresent(id -> { - cfLinks.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, id)); - }); + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); + List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); + List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + EntityId profileId = getProfileId(tenantId, entityId); + + List cfLinks = Stream.concat( + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), + profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() + ).toList(); + cfLinks.forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map attributes = link.getConfiguration().getAttributes(); - Map timeSeries = link.getConfiguration().getTimeSeries(); + Map telemetryKeys = getTelemetryKeysFromLink(link, scope); Map updatedTelemetry = telemetry.stream() - .filter(entry -> attributes.containsValue(entry.getKey()) || timeSeries.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( - entry -> getMappedKey(entry, attributes, timeSeries), + entry -> getMappedKey(entry, telemetryKeys), entry -> entry, (v1, v2) -> v1 )); @@ -368,25 +374,24 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } } catch (Exception e) { - log.trace("Failed to update telemetry entityId: [{}]", entityId, e); + log.trace("Failed to update telemetry.", e); } } - private String getMappedKey(KvEntry entry, Map attributes, Map timeSeries) { - if (entry instanceof AttributeKvEntry) { - return attributes.entrySet().stream() - .filter(attr -> attr.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } else if (entry instanceof TsKvEntry) { - return timeSeries.entrySet().stream() - .filter(ts -> ts.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); - } - return entry.getKey(); + private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { + return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } + + private String getMappedKey(KvEntry entry, Map telemetry) { + return telemetry.entrySet().stream() + .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) + .map(Map.Entry::getKey) + .findFirst() + .orElse(entry.getKey()); } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { @@ -539,17 +544,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { - ReentrantLock lock = entityLocks.computeIfAbsent(entityId, id -> new ReentrantLock()); - lock.lock(); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - try { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); - CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, ctxId -> fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType())); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { if (state.updateState(argumentValues)) { calculatedFieldEntityCtx.setState(state); - states.put(entityCtxId, calculatedFieldEntityCtx); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && @@ -564,12 +566,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> "TS_ROLLING".equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || ("TS_ROLLING".equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) @@ -578,9 +580,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } - } finally { - lock.unlock(); - } + return calculatedFieldEntityCtx; + }); } else { sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); } @@ -605,9 +606,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { try { - String type = calculatedFieldResult.getType(); - TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; - TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; + OutputType type = calculatedFieldResult.getType(); + TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; + TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); if (calculatedFieldIds == null) { calculatedFieldIds = new ArrayList<>(); @@ -649,19 +650,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getType()) { - case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); - case "ATTRIBUTE" -> transformSingleValueArgument( + case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); + case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), - result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); - case "TS_LATEST" -> transformSingleValueArgument( + case TS_LATEST -> transformSingleValueArgument( Futures.transform( timeseriesService.findLatest(tenantId, entityId, argument.getKey()), - result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), + result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); - default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'."); }; } @@ -801,4 +801,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); } + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index dd56405352..b261840bfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -35,7 +35,7 @@ import java.util.List; public interface ArgumentEntry { @JsonIgnore - ArgumentType getType(); + ArgumentEntryType getType(); Object getValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java index 360529a7e9..1a0dfb5ac7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java @@ -15,6 +15,6 @@ */ package org.thingsboard.server.service.cf.ctx.state; -public enum ArgumentType { +public enum ArgumentEntryType { SINGLE_VALUE, TS_ROLLING } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index be7319b930..bc9a421e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -49,8 +49,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry - && singleValueEntry.getVersion() > existingSingleValueEntry.getVersion()) { + } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { +// Long existingVersion = existingSingleValueEntry.getVersion(); +// Long newVersion = singleValueEntry.getVersion(); +// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { +// arguments.put(key, newEntry.copy()); +// } arguments.put(key, newEntry.copy()); } else { arguments.put(key, newEntry.copy()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 20b531f562..8d5080e90f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -32,7 +32,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private long version; + private Long version; public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { @@ -54,8 +54,8 @@ public class SingleValueArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.SINGLE_VALUE; + public ArgumentEntryType getType() { + return ArgumentEntryType.SINGLE_VALUE; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 64de2f8c8a..1118e3af13 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -51,8 +51,8 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public ArgumentType getType() { - return ArgumentType.TS_ROLLING; + public ArgumentEntryType getType() { + return ArgumentEntryType.TS_ROLLING; } @JsonIgnore diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java new file mode 100644 index 0000000000..8479ff37d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private AttributeScope scope; + private List kvEntries; + private List calculatedFieldIds; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java new file mode 100644 index 0000000000..3c28833f31 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; + +import java.util.List; + +public interface CalculatedFieldTelemetryUpdateRequest { + + TenantId getTenantId(); + + EntityId getEntityId(); + + AttributeScope getScope(); + + List getKvEntries(); + + List getCalculatedFieldIds(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java new file mode 100644 index 0000000000..987d899465 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.telemetry; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +@Data +@AllArgsConstructor +public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { + + private TenantId tenantId; + private EntityId entityId; + private List kvEntries; + private List calculatedFieldIds; + + @Override + public AttributeScope getScope() { + return null; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index e35d3cead6..bf3076be5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,6 +50,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -152,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(tenantId, entityId, request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); return saveFuture; } @@ -168,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(request.getTenantId(), request.getEntityId(), request.getCalculatedFieldIds(), request.getEntries()); + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index ba1dfb1fec..5d1467974d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -24,8 +24,10 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -140,7 +142,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -149,7 +151,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java index c5f81cd572..78513c8b74 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java @@ -23,7 +23,9 @@ import java.util.Map; @Data public class CalculatedFieldLinkConfiguration { - private Map attributes = new HashMap<>(); + private Map clientAttributes = new HashMap<>(); + private Map serverAttributes = new HashMap<>(); + private Map sharedAttributes = new HashMap<>(); private Map timeSeries = new HashMap<>(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index f34f5e9cb7..4dac866219 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -24,7 +24,7 @@ public class Argument { private EntityId entityId; private String key; - private String type; + private ArgumentType type; private AttributeScope scope; private String defaultValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java new file mode 100644 index 0000000000..17e2315b52 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum ArgumentType { + + TS_LATEST, ATTRIBUTE, TS_ROLLING + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index ac36991a61..8c86b6c552 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -65,19 +65,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); - for (Map.Entry entry : arguments.entrySet()) { - Argument argument = entry.getValue(); - if (argument.getEntityId().equals(entityId)) { - switch (argument.getType()) { - case "ATTRIBUTE": - linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey()); - break; - case "TS_LATEST", "TS_ROLLING": - linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); - break; - } - } - } + arguments.entrySet().stream() + .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .forEach(entry -> { + Argument argument = entry.getValue(); + String argumentKey = entry.getKey(); + + switch (argument.getType()) { + case ATTRIBUTE -> { + switch (argument.getScope()) { + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + } + } + case TS_LATEST, TS_ROLLING -> + linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + } + }); return linkConfiguration; } @@ -98,7 +103,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityId", entityId.toString()); } argumentNode.put("key", argument.getKey()); - argumentNode.put("type", argument.getType()); + argumentNode.put("type", String.valueOf(argument.getType())); argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); @@ -112,7 +117,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (output != null) { ObjectNode outputNode = configNode.putObject("output"); outputNode.put("name", output.getName()); - outputNode.put("type", output.getType()); + outputNode.put("type", String.valueOf(output.getType())); if (output.getScope() != null) { outputNode.put("scope", String.valueOf(output.getScope())); } @@ -141,7 +146,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } argument.setKey(argumentNode.get("key").asText()); - argument.setType(argumentNode.get("type").asText()); + JsonNode type = argumentNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + argument.setType(ArgumentType.valueOf(type.asText())); + } JsonNode scope = argumentNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { argument.setScope(AttributeScope.valueOf(scope.asText())); @@ -169,7 +177,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (outputNode != null) { Output output = new Output(); output.setName(outputNode.get("name").asText()); - output.setType(outputNode.get("type").asText()); + JsonNode type = outputNode.get("type"); + if (type != null && !type.isNull() && !type.asText().equals("null")) { + output.setType(OutputType.valueOf(type.asText())); + } JsonNode scope = outputNode.get("scope"); if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { output.setScope(AttributeScope.valueOf(scope.asText())); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 46257d1ccc..12cf97338a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.AttributeScope; public class Output { private String name; - private String type; + private OutputType type; private AttributeScope scope; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java new file mode 100644 index 0000000000..c248bc8042 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +public enum OutputType { + + TIME_SERIES, ATTRIBUTES + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index b0870f3dc5..9a0b9222f0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -33,7 +33,9 @@ import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -884,7 +886,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -893,7 +895,7 @@ public class AssetServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 9a1719e715..5a8f7a2383 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -26,8 +26,10 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -153,7 +155,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -162,7 +164,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 6671e0e821..d0ee833261 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -34,7 +34,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -379,7 +381,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -388,7 +390,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index f2f8686bc3..5b060ae145 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1222,7 +1224,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); - argument.setType("TS_LATEST"); + argument.setType(ArgumentType.TS_LATEST); argument.setKey("temperature"); config.setArguments(Map.of("T", argument)); @@ -1231,7 +1233,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Output output = new Output(); output.setName("output"); - output.setType("TS_LATEST"); + output.setType(OutputType.TIME_SERIES); config.setOutput(output); From 8f87caba148aac97973b0d0f212d998957eb5a00 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 30 Dec 2024 16:50:01 +0200 Subject: [PATCH 066/438] implemented logic to handle profile updates even if no cf exists by them --- ...efaultCalculatedFieldExecutionService.java | 11 +++--- .../queue/DefaultTbClusterService.java | 36 ++++--------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 27db9ae290..96cbdfc73b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -542,6 +542,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); + Map argumentsMap = new HashMap<>(argumentValues); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -550,7 +551,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); Consumer performUpdateState = (state) -> { - if (state.updateState(argumentValues)) { + if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); @@ -564,17 +565,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentValues.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentValues.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentValues::putAll) + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) .addListener(() -> performUpdateState.accept(state), calculatedFieldCallbackExecutor); } else { @@ -583,7 +584,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentValues); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index d57a5266bf..6316d1fdb2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -68,7 +68,6 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -150,7 +149,6 @@ public class DefaultTbClusterService implements TbClusterService { private final GatewayNotificationsService gatewayNotificationsService; private final EdgeService edgeService; private final TbTransactionalCache edgeIdServiceIdCache; - private final CalculatedFieldService calculatedFieldService; @Override public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { @@ -393,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); + sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -402,17 +400,10 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); + sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } - private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, false, true); - } - } - @Override public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { onDeviceDeleted(oldTenantId, device, null); @@ -633,13 +624,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -653,29 +644,14 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } - private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId); - boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId); - if (cfExistsByOldProfile || cfExistsByNewProfile) { - sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId); - } - } - - private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) { - boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId); - if (cfExistsByProfile) { - sendProfileEntityEvent(tenantId, entityId, profileId, true, false); - } - } - @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { From 63b79b7242d1c695cbacb46d156495247aa38ba8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 31 Dec 2024 16:15:42 +0200 Subject: [PATCH 067/438] added completable future to prevent reading old value from while updating is in progress --- .../DefaultCalculatedFieldExecutionService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 96cbdfc73b..976139e8bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -94,6 +94,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; @@ -543,13 +544,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); if (tpi.isMyPartition()) { + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + CompletableFuture updateFuture = new CompletableFuture<>(); + Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); @@ -561,6 +566,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } } + updateFuture.complete(null); }; CalculatedFieldState state = calculatedFieldEntityCtx.getState(); @@ -570,7 +576,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -581,6 +586,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { performUpdateState.accept(state); } + + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } + return calculatedFieldEntityCtx; }); } else { From 4ebb68ded68159fcda166e06499bbd75e6bbe53e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 3 Jan 2025 16:00:35 +0200 Subject: [PATCH 068/438] handled profile updates in cluster --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 25 +++++++++++ ...efaultCalculatedFieldExecutionService.java | 44 +++++++++--------- .../queue/DefaultTbClusterService.java | 26 ++++++----- .../queue/DefaultTbCoreConsumerService.java | 38 ++++++++++++++-- .../queue/DefaultTbEdgeConsumerService.java | 8 ++-- .../DefaultTbRuleEngineConsumerService.java | 6 ++- .../processing/AbstractConsumerService.java | 9 ++++ common/proto/src/main/proto/queue.proto | 2 + docker/docker-compose.cluster.yml | 45 +++++++++++++++++++ 10 files changed, 166 insertions(+), 39 deletions(-) create mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index f953e57cc3..ad683c324c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -34,6 +34,8 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index dd2ab3857c..a4077b599b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -141,6 +141,31 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } + + @Override + public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); + calculatedFieldFetchLock.lock(); + try { + List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + if (cfLinks != null && !cfLinks.isEmpty()) { + cfLinks.forEach(link -> { + entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { + if (existingList == null) { + existingList = new ArrayList<>(); + } else if (!(existingList instanceof ArrayList)) { + existingList = new ArrayList<>(existingList); + } + existingList.add(link); + return existingList; + }); + }); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 976139e8bd..2f55de6b77 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,7 +71,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -105,7 +104,6 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; -@TbCoreComponent @Service @Slf4j @RequiredArgsConstructor @@ -117,7 +115,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + // private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -213,8 +211,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedField cf, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); +// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); @@ -313,13 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (calculatedFieldIds != null) { calculatedFieldIds.remove(calculatedFieldId); } - calculatedFieldCache.evict(calculatedFieldId); +// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); - rocksDBService.deleteAll(statesToRemove); +// rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -414,7 +413,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); } - log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); } @Override @@ -423,7 +421,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - + log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); if (proto.getClear()) { clearState(tenantId, calculatedFieldId, entityId); return; @@ -431,7 +429,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List calculatedFieldIds = proto.getCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList(); + .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); @@ -451,8 +449,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -472,15 +470,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); + + List calculatedFieldIds = Stream.concat( calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) ).toList(); + calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -494,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); +// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -558,13 +559,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); +// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); } + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } updateFuture.complete(null); }; @@ -633,6 +635,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldIds.add(calculatedFieldId); TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); + log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); } @@ -715,6 +718,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); } + log.info("Sending calculated field state msg from entityId [{}]", entityId); clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); } @@ -790,11 +794,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); +// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); +// if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); +// } +// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 6316d1fdb2..551b0ddbc2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -391,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -400,7 +400,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true); + handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -563,7 +563,9 @@ public class DefaultTbClusterService implements TbClusterService { || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) - || entityType.equals(EntityType.NOTIFICATION_RULE)) { + || entityType.equals(EntityType.NOTIFICATION_RULE) + || entityType.equals(EntityType.CALCULATED_FIELD) + ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); for (String serviceId : tbCoreServices) { @@ -624,13 +626,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceTypeChanged = !device.getType().equals(old.getType()); if (deviceTypeChanged) { - sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceTypeChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -644,10 +646,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -792,8 +794,8 @@ public class DefaultTbClusterService implements TbClusterService { public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { @@ -809,7 +811,7 @@ public class DefaultTbClusterService implements TbClusterService { pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); } - private void sendEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -822,10 +824,12 @@ public class DefaultTbClusterService implements TbClusterService { builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); } - private void sendProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { + private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); @@ -838,6 +842,8 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setDeleted(deleted); TransportProtos.ProfileEntityMsgProto msg = builder.build(); + + broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 1be25b308f..6baa75b3ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; @@ -87,6 +88,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; @@ -109,6 +111,7 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -181,8 +184,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); + if (added) { + entitiesByProfile.add(entityId); + } else { + entitiesByProfile.remove(entityId); + } + } + private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); @@ -688,12 +718,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); DonAsynchron.withCallback(future, __ -> callback.onSuccess(), t -> { - log.warn("[{}] Failed to process device type updated message for device [{}]", tenantId.getId(), entityId.getId(), t); + log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); callback.onFailure(t); }); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index f219d7ae69..35c7894f2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -91,7 +91,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService future = edgeCtx.getTenantProfileProcessor().processEntityNotification(tenantId, edgeNotificationMsg); case NOTIFICATION_RULE, NOTIFICATION_TARGET, NOTIFICATION_TEMPLATE -> future = edgeCtx.getNotificationEdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case TB_RESOURCE -> future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); - case DOMAIN, OAUTH2_CLIENT -> future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case TB_RESOURCE -> + future = edgeCtx.getResourceProcessor().processEntityNotification(tenantId, edgeNotificationMsg); + case DOMAIN, OAUTH2_CLIENT -> + future = edgeCtx.getOAuth2EdgeProcessor().processEntityNotification(tenantId, edgeNotificationMsg); default -> { future = Futures.immediateFuture(null); log.warn("[{}] Edge event type [{}] is not designed to be pushed to edge", tenantId, type); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 73810949b3..7d4d975cb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -46,6 +46,7 @@ import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; @@ -83,8 +84,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - JwtSettingsService jwtSettingsService) { - super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); + JwtSettingsService jwtSettingsService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.ctx = ctx; this.tbDeviceRpcService = tbDeviceRpcService; this.queueService = queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 95fc8cb843..2aaed13ec1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -43,6 +44,7 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.TbPackCallback; @@ -68,6 +70,7 @@ public abstract class AbstractConsumerService Date: Mon, 6 Jan 2025 14:15:32 +0200 Subject: [PATCH 069/438] changed partititoning implementation --- ...efaultCalculatedFieldExecutionService.java | 180 ++++++++---------- ...CalculatedFieldAttributeUpdateRequest.java | 13 +- ...CalculatedFieldTelemetryUpdateRequest.java | 9 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 9 +- .../DefaultTelemetrySubscriptionService.java | 20 +- .../thingsboard/server/common/msg/TbMsg.java | 32 ++-- common/proto/src/main/proto/queue.proto | 2 +- docker/docker-compose.cluster.yml | 45 ----- .../engine/api/AttributesSaveRequest.java | 10 +- .../engine/api/TimeseriesSaveRequest.java | 10 +- .../engine/telemetry/TbMsgAttributesNode.java | 2 +- .../engine/telemetry/TbMsgTimeseriesNode.java | 2 +- 12 files changed, 145 insertions(+), 189 deletions(-) delete mode 100644 docker/docker-compose.cluster.yml diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 2f55de6b77..f7bcb4e678 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,6 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -98,7 +97,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.stream.Stream; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; @@ -115,7 +113,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - // private final RocksDBService rocksDBService; + private final RocksDBService rocksDBService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -168,35 +166,38 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiCalculatedFieldMap = new HashMap<>(); + Map> tpiTargetEntityMap = new HashMap<>(); for (CalculatedField cf : cfs) { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId()); - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}", - cf.getId(), cf.getTenantId(), e.getMessage()); - continue; - } - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf); + + Consumer resolvePartition = entityId -> { + TopicPartitionInfo tpi; + try { + tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + } + } catch (Exception e) { + log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", + entityId, cf.getTenantId(), e.getMessage()); + } + }; + + EntityId cfEntityId = cf.getEntityId(); + if (isProfileEntity(cfEntityId)) { + calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); + } else { + resolvePartition.accept(cfEntityId); } } - for (var entry : tpiCalculatedFieldMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { + for (var entry : tpiTargetEntityMap.entrySet()) { + for (List partition : Lists.partition(entry.getValue(), 1000)) { log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); var future = calculatedFieldExecutor.submit(() -> { try { - for (CalculatedField cf : partition) { - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId) - .forEach(entityId -> restoreState(cf, entityId)); - } else { - restoreState(cf, cfEntityId); - } + for (CalculatedFieldEntityCtxId ctxId : partition) { + restoreState(ctxId.cfId(), ctxId.entityId()); } } catch (Throwable t) { log.error("Unexpected exception while restoring CalculatedField states", t); @@ -209,17 +210,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(CalculatedField cf, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId()); -// String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + private void restoreState(UUID calculatedFieldId, UUID entityId) { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); - String storedState = null; if (storedState != null) { CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", cf.getId()); + log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId); + log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); } } @@ -238,12 +238,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - if (!tpi.isMyPartition()) { - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null); - log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName()); - callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!")); - } if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); @@ -307,18 +301,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); - Set calculatedFieldIds = partitionedEntities.get(tpi); - if (calculatedFieldIds != null) { - calculatedFieldIds.remove(calculatedFieldId); - } -// calculatedFieldCache.evict(calculatedFieldId); states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); List statesToRemove = states.keySet().stream() .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) .map(JacksonUtil::writeValueAsString) .toList(); -// rocksDBService.deleteAll(statesToRemove); + rocksDBService.deleteAll(statesToRemove); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -345,22 +333,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas try { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); - AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope(); - List telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries(); - List calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); - List cfLinks = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(), - profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty() - ).toList(); - - cfLinks.forEach(link -> { + getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = getTelemetryKeysFromLink(link, scope); - Map updatedTelemetry = telemetry.stream() + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() .filter(entry -> telemetryKeys.containsValue(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), @@ -369,7 +349,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas )); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry); + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } }); } @@ -378,14 +359,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private Map getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) { - return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); - }; - } - private String getMappedKey(KvEntry entry, Map telemetry) { return telemetry.entrySet().stream() .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) @@ -394,7 +367,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .orElse(entry.getKey()); } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List calculatedFieldIds, Map updatedTelemetry) { + private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); @@ -406,12 +379,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds)); + calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); } } - default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds); + default -> + updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); } } @@ -427,14 +401,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return; } - List calculatedFieldIds = proto.getCalculatedFieldsList().stream() + List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) .collect(Collectors.toCollection(ArrayList::new)); Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds); + updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); } @@ -449,9 +423,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId); - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) .forEach(cfId -> clearState(tenantId, cfId, entityId)); @@ -470,18 +441,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId); - - - List calculatedFieldIds = Stream.concat( - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId), - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId) - ).toList(); - calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId)); + getCalculatedFieldLinks(tenantId, entityId, profileId) + .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); } } catch (Exception e) { @@ -490,12 +454,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); states.remove(ctxId); -// rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); } @@ -541,12 +505,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List calculatedFieldIds) { + private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); @@ -559,12 +523,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer performUpdateState = (state) -> { if (state.updateState(argumentsMap)) { calculatedFieldEntityCtx.setState(state); -// rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); + rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); Map arguments = state.getArguments(); boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds); + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); } @@ -599,17 +563,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return calculatedFieldEntityCtx; }); } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap); + sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); } } - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List calculatedFieldIds) { + private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); Futures.addCallback(resultFuture, new FutureCallback<>() { @Override public void onSuccess(CalculatedFieldResult result) { if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds); + pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); } } @@ -620,20 +584,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, MoreExecutors.directExecutor()); } - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List calculatedFieldIds) { + private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (calculatedFieldIds == null) { - calculatedFieldIds = new ArrayList<>(); - } - if (calculatedFieldIds.contains(calculatedFieldId)) { + if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); } + List calculatedFieldIds = previousCalculatedFieldIds != null + ? new ArrayList<>(previousCalculatedFieldIds) + : new ArrayList<>(); calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); } catch (Exception e) { @@ -641,6 +605,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + if (profileId != null) { + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + } + return links; + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -704,13 +676,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List calculatedFieldIds, Map argumentValues) { + private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); if (argumentValues != null) { argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); } - if (calculatedFieldIds != null) { - calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields( + if (previousCalculatedFieldIds != null) { + previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) @@ -794,11 +766,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { -// String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); -// if (stateStr == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); -// } -// return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); + if (stateStr == null) { + return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); + } + return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 8479ff37d7..c56217b2ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -33,6 +35,15 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private EntityId entityId; private AttributeScope scope; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; + + @Override + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return switch (scope) { + case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); + case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); + case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + }; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 3c28833f31..98062a08db 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; +import java.util.Map; public interface CalculatedFieldTelemetryUpdateRequest { @@ -29,10 +30,10 @@ public interface CalculatedFieldTelemetryUpdateRequest { EntityId getEntityId(); - AttributeScope getScope(); - List getKvEntries(); - List getCalculatedFieldIds(); + List getPreviousCalculatedFieldIds(); + + Map getTelemetryKeysFromLink(CalculatedFieldLink link); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 987d899465..bd2161dca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; +import java.util.Map; @Data @AllArgsConstructor @@ -32,11 +33,11 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private TenantId tenantId; private EntityId entityId; private List kvEntries; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; @Override - public AttributeScope getScope() { - return null; + public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + return link.getConfiguration().getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index bf3076be5c..99cbd89494 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); return saveFuture; } @@ -170,7 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds())); + addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); } @Override @@ -243,7 +243,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer .onlyLatest(true) .callback(new FutureCallback<>() { @Override - public void onSuccess(@Nullable Void tmp) {} + public void onSuccess(@Nullable Void tmp) { + } @Override public void onFailure(Throwable t) { @@ -342,4 +343,17 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } + protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { + Futures.addCallback(saveFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable T result) { + callback.accept(result); + } + + @Override + public void onFailure(Throwable t) { + } + }, tsCallBackExecutor); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 0805175f77..9d99f38c60 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -35,10 +35,10 @@ import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import java.io.Serializable; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; /** * Created by ashvayka on 13.01.18. @@ -67,7 +67,7 @@ public final class TbMsg implements Serializable { private final UUID correlationId; private final Integer partition; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; @Getter(value = AccessLevel.NONE) @JsonIgnore @@ -117,7 +117,7 @@ public final class TbMsg implements Serializable { } private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, - RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { + RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List previousCalculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) { this.id = id != null ? id : UUID.randomUUID(); this.queueName = queueName; if (ts > 0) { @@ -144,7 +144,9 @@ public final class TbMsg implements Serializable { this.ruleNodeId = ruleNodeId; this.correlationId = correlationId; this.partition = partition; - this.calculatedFieldIds = calculatedFieldIds; + this.previousCalculatedFieldIds = previousCalculatedFieldIds != null + ? new CopyOnWriteArrayList<>(previousCalculatedFieldIds) + : new CopyOnWriteArrayList<>(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); } @@ -192,8 +194,8 @@ public final class TbMsg implements Serializable { builder.setPartition(msg.getPartition()); } - if (msg.getCalculatedFieldIds() != null) { - for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) { + if (msg.getPreviousCalculatedFieldIds() != null) { + for (CalculatedFieldId calculatedFieldId : msg.getPreviousCalculatedFieldIds()) { MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) @@ -216,7 +218,7 @@ public final class TbMsg implements Serializable { RuleNodeId ruleNodeId = null; UUID correlationId = null; Integer partition = null; - List calculatedFieldIds = new ArrayList<>(); + List calculatedFieldIds = new CopyOnWriteArrayList<>(); if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); } @@ -274,6 +276,7 @@ public final class TbMsg implements Serializable { /** * Checks if the message is still valid for processing. May be invalid if the message pack is timed-out or canceled. + * * @return 'true' if message is valid for processing, 'false' otherwise. */ public boolean isValid() { @@ -368,7 +371,7 @@ public final class TbMsg implements Serializable { protected RuleNodeId ruleNodeId; protected UUID correlationId; protected Integer partition; - protected List calculatedFieldIds; + protected List previousCalculatedFieldIds; protected TbMsgProcessingCtx ctx; protected TbMsgCallback callback; @@ -390,7 +393,7 @@ public final class TbMsg implements Serializable { this.ruleNodeId = tbMsg.ruleNodeId; this.correlationId = tbMsg.correlationId; this.partition = tbMsg.partition; - this.calculatedFieldIds = tbMsg.calculatedFieldIds; + this.previousCalculatedFieldIds = tbMsg.previousCalculatedFieldIds; this.ctx = tbMsg.ctx; this.callback = tbMsg.callback; } @@ -413,8 +416,7 @@ public final class TbMsg implements Serializable { /** *

Deprecated: This should only be used when you need to specify a custom message type that doesn't exist in the {@link TbMsgType} enum. * Prefer using {@link #type(TbMsgType)} instead. - * - * */ + */ @Deprecated public TbMsgBuilder type(String type) { this.type = type; @@ -482,8 +484,8 @@ public final class TbMsg implements Serializable { return this; } - public TbMsgBuilder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public TbMsgBuilder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -498,7 +500,7 @@ public final class TbMsg implements Serializable { } public TbMsg build() { - return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback); + return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, previousCalculatedFieldIds, ctx, callback); } public String toString() { @@ -506,7 +508,7 @@ public final class TbMsg implements Serializable { ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + - ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds + + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", previousCalculatedFields=" + this.previousCalculatedFieldIds + ", ctx=" + this.ctx + ", callback=" + this.callback + ")"; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1c4927c422..9cba62eecc 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -818,7 +818,7 @@ message CalculatedFieldStateMsgProto { int64 entityIdMSB = 6; int64 entityIdLSB = 7; bool clear = 8; - repeated CalculatedFieldIdProto calculatedFields = 9; + repeated CalculatedFieldIdProto previousCalculatedFields = 9; map arguments = 10; } diff --git a/docker/docker-compose.cluster.yml b/docker/docker-compose.cluster.yml deleted file mode 100644 index 4c74238a10..0000000000 --- a/docker/docker-compose.cluster.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright © 2016-2024 The Thingsboard Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -version: '3.0' - -services: - kafka: - restart: always - image: "bitnami/kafka:3.7.0" - ports: - - "9092:9092" - env_file: - - kafka.env - depends_on: - - zookeeper - zookeeper: - restart: always - image: "zookeeper:3.8.0" - ports: - - "2181" - environment: - ZOO_MY_ID: 1 - ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - ZOO_ADMINSERVER_ENABLED: "false" - redis: - restart: always - image: bitnami/redis:7.2 - environment: - # ALLOW_EMPTY_PASSWORD is recommended only for development. - ALLOW_EMPTY_PASSWORD: "yes" - ports: - - '6379:6379' diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 9747a6033e..406b2d0d5c 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -41,7 +41,7 @@ public class AttributesSaveRequest { private final AttributeScope scope; private final List entries; private final boolean notifyDevice; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -55,7 +55,7 @@ public class AttributesSaveRequest { private AttributeScope scope; private List entries; private boolean notifyDevice = true; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; private FutureCallback callback; Builder() {} @@ -103,8 +103,8 @@ public class AttributesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -128,7 +128,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 12afa2d939..95eb788e5f 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -41,7 +41,7 @@ public class TimeseriesSaveRequest { private final long ttl; private final boolean saveLatest; private final boolean onlyLatest; - private final List calculatedFieldIds; + private final List previousCalculatedFieldIds; private final FutureCallback callback; public static Builder builder() { @@ -58,7 +58,7 @@ public class TimeseriesSaveRequest { private FutureCallback callback; private boolean saveLatest = true; private boolean onlyLatest; - private List calculatedFieldIds; + private List previousCalculatedFieldIds; Builder() {} @@ -106,8 +106,8 @@ public class TimeseriesSaveRequest { return this; } - public Builder calculatedFieldIds(List calculatedFieldIds) { - this.calculatedFieldIds = calculatedFieldIds; + public Builder previousCalculatedFieldIds(List previousCalculatedFieldIds) { + this.previousCalculatedFieldIds = previousCalculatedFieldIds; return this; } @@ -131,7 +131,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index 20d0dda42f..ae2fce6575 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -125,7 +125,7 @@ public class TbMsgAttributesNode implements TbNode { .scope(scope) .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 386e56320a..89f7844313 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -112,7 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode { .entries(tsKvEntryList) .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) - .calculatedFieldIds(msg.getCalculatedFieldIds()) + .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From 7d8a76ce7f8e1ca15e08709bb2fcba8446578fbf Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 6 Jan 2025 15:32:23 +0200 Subject: [PATCH 070/438] Draft of the review --- .../cf/DefaultCalculatedFieldCache.java | 10 ++++------ .../AbstractSubscriptionService.java | 9 +++++++-- .../DefaultTelemetrySubscriptionService.java | 20 +++++-------------- common/proto/src/main/proto/queue.proto | 7 +++++++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index a4077b599b..c4293b9edc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -122,16 +122,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - if (cfLinks != null) { - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } + entityIdCalculatedFieldLinks.put(entityId, cfLinks); + log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index e26ede2bae..0bb8f76398 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.service.subscription.SubscriptionManagerService; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Consumer; @@ -99,7 +100,11 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList } protected void addWsCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { + addCallback(saveFuture, callback, wsCallBackExecutor); + } + + protected void addCallback(ListenableFuture saveFuture, Consumer callback, Executor executor) { + Futures.addCallback(saveFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable T result) { callback.accept(result); @@ -108,7 +113,7 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList @Override public void onFailure(Throwable t) { } - }, wsCallBackExecutor); + }, executor); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 99cbd89494..f873e48774 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,7 +154,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds()))); + // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. + //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return saveFuture; } @@ -170,7 +172,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCalculatedFieldCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds()))); + //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); } @Override @@ -343,17 +346,4 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer }; } - protected void addCalculatedFieldCallback(ListenableFuture saveFuture, Consumer callback) { - Futures.addCallback(saveFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable T result) { - callback.accept(result); - } - - @Override - public void onFailure(Throwable t) { - } - }, tsCallBackExecutor); - } - } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9cba62eecc..56f347f311 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -809,6 +809,13 @@ message ProfileEntityMsgProto { bool deleted = 10; } +message ToServerB { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + repeated CfIdEntityIdPair links = 3; + value = 4; +} + message CalculatedFieldStateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; From e2ac2708b658c3802c9f15f9825bb621a3730f85 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 6 Jan 2025 17:37:33 +0200 Subject: [PATCH 071/438] changed CF links config --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 37 ++++++++++++++----- ...efaultCalculatedFieldExecutionService.java | 28 +++++++++++++- ...CalculatedFieldAttributeUpdateRequest.java | 21 +++++++---- ...CalculatedFieldTelemetryUpdateRequest.java | 4 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 16 +++++--- .../DefaultTelemetrySubscriptionService.java | 7 +--- .../server/dao/cf/CalculatedFieldService.java | 2 + .../BaseCalculatedFieldConfiguration.java | 17 +++++---- .../dao/cf/BaseCalculatedFieldService.java | 7 ++++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 ++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 13 files changed, 114 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ad683c324c..aa25565a34 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -30,6 +30,8 @@ public interface CalculatedFieldCache { CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c4293b9edc..40d569b040 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -56,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final DeviceService deviceService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); @@ -65,10 +66,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Getter private int initFetchPackSize; - @PostConstruct public void init() { - // to discuss: fetch on start or fetch on demand PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); @@ -97,19 +96,37 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return calculatedField; } + @Override + public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + List cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + calculatedFieldFetchLock.lock(); + try { + cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); + entityIdCalculatedFields.put(entityId, cfs); + log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); + return cfs; + } + @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cfLinks != null) { - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); @@ -139,7 +156,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } - @Override public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); @@ -225,6 +241,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f7bcb4e678..c0f7acf0a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -139,6 +140,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(() -> rocksDBService.getAll() + .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); } @PreDestroy @@ -337,11 +340,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); + // process by profile + if (profileId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, telemetryKeys), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); + } + }); + } + + // process by links getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), entry -> entry, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index c56217b2ce..25d2f57bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -37,12 +36,20 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.scope = request.getScope(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { return switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 98062a08db..29ee899ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLink link); + Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index bd2161dca1..6225286631 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -35,9 +34,16 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { - return link.getConfiguration().getTimeSeries(); + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { + return linkConfiguration.getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..8773564e5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,9 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)), tsCallBackExecutor); return saveFuture; } @@ -172,8 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 1e64fdac60..3a508a5c08 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -38,6 +38,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8c86b6c552..c3c4e32507 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -68,19 +68,22 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel arguments.entrySet().stream() .filter(entry -> entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument argument = entry.getValue(); + Argument tergetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (argument.getType()) { + switch (tergetArgument.getType()) { case ATTRIBUTE -> { - switch (argument.getScope()) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + switch (tergetArgument.getScope()) { + case CLIENT_SCOPE -> + linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + case SERVER_SCOPE -> + linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + case SHARED_SCOPE -> + linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); } }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0849414a0e..e650aec35e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -98,6 +98,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); } + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 5b3bcc2750..39663d0afc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -31,6 +31,8 @@ public interface CalculatedFieldDao extends Dao { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 9aa0aee428..816fa1546c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -28,6 +29,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index e3762f6157..20081299e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -55,6 +55,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 03c3341265724341aea3668f6f376a7e90d842f3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 12:39:49 +0200 Subject: [PATCH 072/438] added logic to send msgs to RE when not my partition --- .../server/controller/BaseController.java | 7 - .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 252 ++++++++++++++---- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 +- ...CalculatedFieldAttributeUpdateRequest.java | 6 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 6 +- .../TbRuleEngineQueueConsumerManager.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 14 +- .../server/common/util/ProtoUtils.java | 133 +++++++++ common/proto/src/main/proto/queue.proto | 29 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + 11 files changed, 386 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 4987096d17..139c61d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -366,9 +366,6 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; - @Autowired - protected CalculatedFieldService calculatedFieldService; - @Autowired protected NotificationTargetService notificationTargetService; @@ -998,10 +995,6 @@ public abstract class BaseController { return null; } - protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { - return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); - } - protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e4b0a7ca1e..6d1d459b9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -25,6 +25,8 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0f7acf0a3..0de3136439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,9 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; @@ -51,6 +53,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -67,6 +70,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -80,7 +84,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -102,6 +108,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -177,8 +184,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi; try { tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } } catch (Exception e) { log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", @@ -213,7 +220,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(UUID calculatedFieldId, UUID entityId) { + private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); @@ -232,7 +239,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } @Override @@ -243,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); + onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -293,7 +300,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { callback.onSuccess(); shouldReinit = false; @@ -301,12 +308,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -334,61 +341,147 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - EntityId profileId = getProfileId(tenantId, entityId); - - // process by profile - if (profileId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); - } + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + Map> tpiStatesToUpdate = new HashMap<>(); + + updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); + updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } + } - // process by links - getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + updateTelemetryForEntity(request, request.getEntityId(), tpiStates); + } + + private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { + updateTelemetryForEntity(request, profileId, tpiStates); + } + + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + if (targetEntity != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + }); + } + } else { + List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + }); + } + } + + private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + + private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + }); + } else { + updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); } }); - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); + } + + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, + EntityId entityId, + CalculatedFieldId calculatedFieldId, + CalculatedFieldTelemetryUpdateRequest request, + CalculatedFieldLinkConfiguration linkConfiguration) { + Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } } - private String getMappedKey(KvEntry entry, Map telemetry) { - return telemetry.entrySet().stream() - .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); + private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { + return kvEntries.stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), + entry -> entry, + (v1, v2) -> v1 + )); + } + + @Override + public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + + proto.getLinksList().forEach(ctxIdProto -> { + EntityId entityId = EntityIdFactory.getByTypeAndUuid( + ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + CalculatedFieldTelemetryUpdateRequest request = attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + + onTelemetryUpdate(request); + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { @@ -481,7 +574,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { @@ -537,7 +630,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); @@ -777,6 +870,57 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( + CalculatedFieldTelemetryUpdateRequest request, List links + ) { + TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); + + builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + + for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { + builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); + } + + if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { + builder.setScope(attributeUpdateRequest.getScope().name()); + } + + for (KvEntry entry : request.getKvEntries()) { + TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); + if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { + telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); + } + if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { + telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); + } + builder.addUpdatedTelemetry(telemetryBuilder.build()); + } + + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index f7c451efee..5fb90a3e46 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.cf.ctx; -import java.util.UUID; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; -public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 25d2f57bd6..a83cc0fc25 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; @@ -22,18 +23,19 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; private AttributeScope scope; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 6225286631..507daf386e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,23 +15,25 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..243a3adbf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -34,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -63,14 +64,18 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument tergetArgument = entry.getValue(); + Argument targetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (tergetArgument.getType()) { + switch (targetArgument.getType()) { case ATTRIBUTE -> { - switch (tergetArgument.getScope()) { + switch (targetArgument.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); } }); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index ec17914fd8..d332bac64f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -58,12 +59,14 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -627,6 +630,136 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static KvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { + if (telemetryProto.hasAttrKv()) { + return fromProto(telemetryProto.getAttrKv().getValue()); + } else if (telemetryProto.hasTsKv()) { + return fromProto(telemetryProto.getTsKv()); + } else { + throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); + } + } + + public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { + TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); + builder.setAttributeKey(key); + switch (scope) { + case CLIENT_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); + break; + case SERVER_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); + break; + case SHARED_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); + break; + default: + throw new IllegalArgumentException("Unsupported attribute scope: " + scope); + } + return builder.build(); + } + + public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { + return TransportProtos.AttributeKvProto.newBuilder() + .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) + .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) + .build(); + } + + public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { + TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); + builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setHasV(true) + .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setHasV(true) + .setLongV(attributeKvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setHasV(true) + .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setHasV(true) + .setStringV(attributeKvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setHasV(true) + .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); + break; + default: + builder.setHasV(false); + throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); + } + if (attributeKvEntry.getKey() != null) { + builder.setKey(attributeKvEntry.getKey()); + } + if (attributeKvEntry.getVersion() != null) { + builder.setVersion(attributeKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + return TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)) + .setVersion(tsKvEntry.getVersion()) + .build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 56f347f311..685ca47719 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,6 +183,18 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } +message AttributeKvProto { + AttributeKey key = 1; + AttributeValueProto value = 2; +} + +message TelemetryProto { + oneof proto { + AttributeKvProto attrKv = 1; + TsKvProto tsKv = 2; + } +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -809,11 +821,21 @@ message ProfileEntityMsgProto { bool deleted = 10; } -message ToServerB { +message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CfIdEntityIdPair links = 3; - value = 4; + repeated CalculatedFieldEntityCtxIdProto links = 3; + repeated CalculatedFieldIdProto previousCalculatedFields = 4; + string scope = 5; + repeated TelemetryProto updatedTelemetry = 6; +} + +message CalculatedFieldEntityCtxIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; } message CalculatedFieldStateMsgProto { @@ -1655,6 +1677,7 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; + TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; } message ToRuleEngineNotificationMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index e650aec35e..36bc3d038a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -247,6 +247,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); return cfConfig.getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) .map(referencedEntityId -> { CalculatedFieldLink link = new CalculatedFieldLink(); link.setTenantId(tenantId); From 46180e33d70761ea18f93238420e8ed1cf97f31e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 17:20:20 +0200 Subject: [PATCH 073/438] cache refactoring --- .../service/cf/CalculatedFieldCache.java | 16 +- .../cf/DefaultCalculatedFieldCache.java | 132 +++++++-------- ...efaultCalculatedFieldExecutionService.java | 156 ++++++++++-------- .../processing/AbstractConsumerService.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 21 +++ .../CalculatedFieldConfiguration.java | 7 + common/proto/src/main/proto/queue.proto | 11 +- .../dao/cf/BaseCalculatedFieldService.java | 18 +- .../dao/sql/cf/CalculatedFieldRepository.java | 3 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 2 +- 10 files changed, 195 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index aa25565a34..7394c95f08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -28,20 +28,24 @@ import java.util.Set; public interface CalculatedFieldCache { - CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void evict(CalculatedFieldId calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 40d569b040..71fc3eff67 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -70,98 +71,44 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + calculatedFields.values().forEach(cf -> + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + ); } @Override - public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedFieldFetchLock.lock(); - try { - calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (calculatedField != null) { - calculatedFields.put(calculatedFieldId, calculatedField); - log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); - return calculatedField; + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); } @Override - public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - List cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - calculatedFieldFetchLock.lock(); - try { - cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); - entityIdCalculatedFields.put(entityId, cfs); - log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); - return cfs; + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { - List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); } @Override - public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); calculatedFieldFetchLock.lock(); try { - List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + List cfLinks = getCalculatedFieldLinks(calculatedFieldId); if (cfLinks != null && !cfLinks.isEmpty()) { cfLinks.forEach(link -> { entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { @@ -181,14 +128,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, ctx); @@ -236,6 +183,42 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void evict(CalculatedFieldId calculatedFieldId) { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); @@ -243,7 +226,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFieldLinks.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); - entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0de3136439..dd11c799c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -253,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -263,7 +264,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -297,7 +298,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -345,17 +346,27 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); - Map> tpiStatesToUpdate = new HashMap<>(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); - updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); - updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (tpi.isMyPartition()) { - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - }); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + }); + } + } else { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -363,30 +374,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - updateTelemetryForEntity(request, request.getEntityId(), tpiStates); - } - - private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { - updateTelemetryForEntity(request, profileId, tpiStates); - } - - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - if (targetEntity != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); - }); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + if (cfTargetEntityId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); }); } } @@ -405,14 +400,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { @@ -451,33 +446,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(request); + return; + } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = EntityIdFactory.getByTypeAndUuid( - ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - CalculatedFieldTelemetryUpdateRequest request = attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - onTelemetryUpdate(request); + CalculatedFieldLinkConfiguration linkConfiguration + = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); + + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); @@ -486,8 +470,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -524,7 +508,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); @@ -559,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(tenantId, entityId, profileId) + getCalculatedFieldLinks(entityId, profileId) .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -585,7 +569,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -722,10 +706,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); } return links; } @@ -870,13 +854,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { + return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); + } + + ; + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(request.getEntityId().getEntityType().name()) + .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -904,6 +897,31 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return builder.build(); } + private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + return attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 2aaed13ec1..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -194,7 +194,9 @@ public abstract class AbstractConsumerService buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + link.setConfiguration(getReferencedEntityConfig(referencedEntityId)); + return link; + } + @Override public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { ObjectNode configNode = mapper.createObjectNode(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..ac94ade134 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -20,9 +20,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; @@ -57,4 +60,8 @@ public interface CalculatedFieldConfiguration { @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 685ca47719..8a7c2d8c03 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -824,10 +824,13 @@ message ProfileEntityMsgProto { message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CalculatedFieldEntityCtxIdProto links = 3; - repeated CalculatedFieldIdProto previousCalculatedFields = 4; - string scope = 5; - repeated TelemetryProto updatedTelemetry = 6; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldEntityCtxIdProto links = 6; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + string scope = 8; + repeated TelemetryProto updatedTelemetry = 9; } message CalculatedFieldEntityCtxIdProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 36bc3d038a..9c81d91f64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -240,23 +239,8 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - List links = buildCalculatedFieldLinks(tenantId, calculatedField); + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); - return cfConfig.getReferencedEntities().stream() - .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) - .map(referencedEntityId -> { - CalculatedFieldLink link = new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(referencedEntityId); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); - return link; - }) - .collect(Collectors.toList()); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 816fa1546c..bed6f2d3a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -29,7 +28,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); - List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 20081299e8..cdcffdd440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -57,7 +57,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); } @Override From 5203ef7422f32a0219a64470298e774c8b683ab8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:45:42 +0200 Subject: [PATCH 074/438] added calculated field state service --- .../service/cf/CalculatedFieldCache.java | 2 - .../cf/CalculatedFieldExecutionService.java | 2 - .../cf/DefaultCalculatedFieldCache.java | 51 +-- ...efaultCalculatedFieldExecutionService.java | 325 ++++++------------ .../server/service/cf/RocksDBService.java | 13 - .../queue/DefaultTbCoreConsumerService.java | 14 - .../server/common/util/ProtoUtils.java | 40 --- common/proto/src/main/proto/queue.proto | 43 +-- 8 files changed, 119 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 7394c95f08..bf5dc8d42f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,8 +36,6 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d1d459b9b..8ba1f6dfed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -27,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 71fc3eff67..f762ae3530 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,12 +36,12 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -72,14 +72,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); } @@ -90,41 +90,17 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); - } - - @Override - public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); - calculatedFieldFetchLock.lock(); - try { - List cfLinks = getCalculatedFieldLinks(calculatedFieldId); - if (cfLinks != null && !cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { - if (existingList == null) { - existingList = new ArrayList<>(); - } else if (!(existingList instanceof ArrayList)) { - existingList = new ArrayList<>(existingList); - } - existingList.add(link); - return existingList; - }); - }); - } - } finally { - calculatedFieldFetchLock.unlock(); - } + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override @@ -192,7 +168,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFields.put(calculatedFieldId, calculatedField); - entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -200,7 +176,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { configuration.getReferencedEntities().stream() .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) .forEach(referencedEntityId -> { - entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { @@ -210,13 +186,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); - try { - evict(calculatedFieldId); - addCalculatedField(tenantId, calculatedFieldId); - } finally { - calculatedFieldFetchLock.unlock(); - } + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd11c799c2..e54c1bd5c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -107,8 +108,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; -import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @@ -122,7 +121,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -148,8 +147,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> rocksDBService.getAll() - .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); + scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -223,10 +221,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - if (storedState != null) { - CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + if (restoredCtx != null) { states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { @@ -251,18 +248,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; } } if (cf != null) { + calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { @@ -312,12 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); + states.keySet().removeIf(ctxId -> { + if (ctxId.cfId().equals(calculatedFieldId)) { + stateService.removeState(ctxId); + return true; + } + return false; + }); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -366,7 +367,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -386,20 +386,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); - if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); @@ -411,14 +397,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, link, tpiStates); }); } else { - updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, link, tpiStates); } }); } + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, @@ -490,31 +490,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); - if (proto.getClear()) { - clearState(tenantId, calculatedFieldId, entityId); - return; - } - - List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .collect(Collectors.toCollection(ArrayList::new)); - Map argumentsMap = proto.getArgumentsMap().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); - } catch (Exception e) { - log.trace("Failed to process calculated field update state msg: [{}]", proto, e); - } - } - @Override public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { @@ -522,12 +497,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> clearState(tenantId, cfId, entityId)); - initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } else { + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); + } } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -539,37 +517,39 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(entityId, profileId) - .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntityByProfile(entityId, profileId, callback); + } } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } + + } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } } - private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - } + private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + states.remove(ctxId); + stateService.removeState(ctxId); } - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) + private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -607,65 +587,58 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - CompletableFuture updateFuture = new CompletableFuture<>(); + CompletableFuture updateFuture = new CompletableFuture<>(); - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentsMap)) { + calculatedFieldEntityCtx.setState(state); + stateService.persistState(entityCtxId, calculatedFieldEntityCtx); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } - updateFuture.complete(null); - }; + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + } + updateFuture.complete(null); + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + if (!allKeysPresent || requiresTsRollingUpdate) { + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } - return calculatedFieldEntityCtx; - }); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); - } + return calculatedFieldEntityCtx; + }); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { @@ -706,14 +679,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); - if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); - } - return links; - } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -777,89 +742,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); - } - if (previousCalculatedFieldIds != null) { - previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( - TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build() - )); - } - - log.info("Sending calculated field state msg from entityId [{}]", entityId); - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) - .setClear(true) - .build(); - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); - } - - private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( - TenantId tenantId, - CalculatedFieldId calculatedFieldId, - EntityId entityId - ) { - return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - } - - private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { - TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> - tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) - ); - argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - argumentProtoBuilder.setSingleValue( - TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(toObjectProto(singleValueArgumentEntry.getValue())) - .build() - ); - } - - return argumentProtoBuilder.build(); - } - - private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> - tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) - ); - return tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); } - ; - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { @@ -952,11 +838,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { + CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); + + if (state == null) { return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + return state; } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index d6b2980042..3aed65eced 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -27,7 +26,6 @@ import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.List; import java.util.Map; @Service @@ -59,17 +57,6 @@ public class RocksDBService { } } - public void deleteAll(List keys) { - try (WriteBatch batch = new WriteBatch()) { - for (String key : keys) { - batch.delete(key.getBytes(StandardCharsets.UTF_8)); - } - db.write(writeOptions, batch); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - public String get(String key) { try { byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6baa75b3ef..c598540ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -326,8 +326,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index d332bac64f..073f47d59b 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1316,46 +1316,6 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.ObjectProto toObjectProto(Object value) { - if (value == null) { - throw new IllegalArgumentException("Cannot convert null to ObjectProto"); - } - - TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); - - if (value instanceof String) { - builder.setStringValue((String) value); - } else if (value instanceof Integer) { - builder.setIntValue((Integer) value); - } else if (value instanceof Long) { - builder.setLongValue((Long) value); - } else if (value instanceof Double) { - builder.setDoubleValue((Double) value); - } else if (value instanceof Boolean) { - builder.setBoolValue((Boolean) value); - } else { - throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); - } - - return builder.build(); - } - - public static Object fromObjectProto(TransportProtos.ObjectProto proto) { - try { - return switch (proto.getValueCase()) { - case STRINGVALUE -> proto.getStringValue(); - case INTVALUE -> proto.getIntValue(); - case LONGVALUE -> proto.getLongValue(); - case DOUBLEVALUE -> proto.getDoubleValue(); - case BOOLVALUE -> proto.getBoolValue(); - case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", proto, e); - return null; - } - } - private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8a7c2d8c03..1036d5ba67 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,51 +841,11 @@ message CalculatedFieldEntityCtxIdProto { int64 entityIdLSB = 5; } -message CalculatedFieldStateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - string entityType = 5; - int64 entityIdMSB = 6; - int64 entityIdLSB = 7; - bool clear = 8; - repeated CalculatedFieldIdProto previousCalculatedFields = 9; - map arguments = 10; -} - message CalculatedFieldIdProto { int64 calculatedFieldIdMSB = 1; int64 calculatedFieldIdLSB = 2; } -message ArgumentEntryProto { - oneof entry_type { - TsRollingProto tsRecords = 1; - SingleValueProto singleValue = 2; - } -} - -message TsRollingProto { - map tsRecords = 1; -} - -message SingleValueProto { - int64 ts = 1; - ObjectProto value = 2; - int64 version = 3; -} - -message ObjectProto { - oneof value { - string stringValue = 1; - int32 intValue = 2; - int64 longValue = 3; - double doubleValue = 4; - bool boolValue = 5; - } -} - //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1632,7 +1592,6 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; - CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1681,6 +1640,8 @@ message ToRuleEngineMsg { repeated string relationTypes = 4; string failureMessage = 5; TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; + ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { From 77e99d15df2c2a3f412c48fce87151d2a473be7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:51:17 +0200 Subject: [PATCH 075/438] added service files --- .../cf/ctx/CalculatedFieldStateService.java | 30 +++++++++ .../cf/ctx/state/RocksDBStateService.java | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java new file mode 100644 index 0000000000..8bc5756f4e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import java.util.Map; + +public interface CalculatedFieldStateService { + + Map restoreStates(); + + CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + + void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + + void removeState(CalculatedFieldEntityCtxId ctxId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java new file mode 100644 index 0000000000..db8950804c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public class RocksDBStateService implements CalculatedFieldStateService { + + private final RocksDBService rocksDBService; + + @Override + public Map restoreStates() { + return rocksDBService.getAll().entrySet().stream() + .collect(Collectors.toMap( + entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + )); + } + + @Override + public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .orElse(null); + } + + @Override + public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId) { + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + +} From 6611f017c7eae1efb05b7bcf0bdf09d17b13ad34 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Jan 2025 16:58:26 +0200 Subject: [PATCH 076/438] changed Argument structure --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 8 ++ ...efaultCalculatedFieldExecutionService.java | 125 ++++++------------ .../cf/ctx/state/CalculatedFieldCtx.java | 15 ++- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 25 ++++ ...CalculatedFieldTelemetryUpdateRequest.java | 3 + ...alculatedFieldTimeSeriesUpdateRequest.java | 31 +++++ .../TbRuleEngineQueueConsumerManager.java | 5 + .../CalculatedFieldControllerTest.java | 2 +- .../data/cf/configuration/Argument.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 49 +++---- .../cf/configuration/ReferencedEntityKey.java | 30 +++++ .../dao/model/sql/CalculatedFieldEntity.java | 3 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 18 files changed, 195 insertions(+), 122 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index bf5dc8d42f..8730aeeedf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -38,6 +38,8 @@ public interface CalculatedFieldCache { CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index f762ae3530..868001d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -61,6 +61,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @@ -126,6 +127,13 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return ctx; } + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .toList(); + } + @Override public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { Set entities = profileEntities.get(entityProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e54c1bd5c9..7c17d54649 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -273,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -341,30 +339,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { try { - EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + TenantId tenantId = request.getTenantId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); } @@ -375,13 +373,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } } @@ -393,56 +390,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); }); } else { - processCalculatedFieldLink(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); } }); } - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - - private void mapAndProcessUpdatedTelemetry(TenantId tenantId, - EntityId entityId, - CalculatedFieldId calculatedFieldId, - CalculatedFieldTelemetryUpdateRequest request, - CalculatedFieldLinkConfiguration linkConfiguration) { - Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); } } - private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { - return kvEntries.stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), - entry -> entry, - (v1, v2) -> v1 - )); - } - @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { @@ -454,40 +427,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - CalculatedFieldLinkConfiguration linkConfiguration - = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); - - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - } - default -> - updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); - } + updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -533,8 +492,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } - - } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } @@ -616,11 +573,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) @@ -696,7 +653,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); + EntityId argumentEntityId = argument.getRefEntityId(); EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; @@ -704,17 +661,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { + return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); case TS_LATEST -> transformSingleValueArgument( Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); }; @@ -736,7 +693,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); @@ -826,7 +783,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); + String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index d54a3220ed..e17a1a61f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -22,13 +22,16 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { @@ -38,7 +41,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final List argKeys; + private final Map, String> referencedEntityKeys; + private final List argNames; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -51,7 +55,12 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); - this.argKeys = new ArrayList<>(arguments.keySet()); + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; @@ -69,7 +78,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - argKeys.toArray(String[]::new) + argNames.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index de7c514786..0421055fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = ctx.getArgKeys().stream() + Object[] args = ctx.getArgNames().stream() .map(key -> arguments.get(key).getValue()) .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index a83cc0fc25..6050370fd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -20,11 +20,16 @@ import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,4 +60,24 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel }; } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + + String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + + if (argName != null) { + mappedKvEntries.put(argName, entry); + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 29ee899ec9..f85117dc41 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; import java.util.Map; @@ -36,4 +37,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); + Map getMappedTelemetry(CalculatedFieldCtx ctx); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 507daf386e..646145a46e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -19,11 +19,16 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,4 +53,30 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return linkConfiguration.getTimeSeries(); } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + + if (argTsLatestName != null) { + mappedKvEntries.put(argTsLatestName, entry); + } else { + ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + + if (argTsRollingName != null) { + mappedKvEntries.put(argTsRollingName, entry); + } + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 243a3adbf7..83ab90e3bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -179,6 +180,10 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager getReferencedEntities() { return arguments.values().stream() - .map(Argument::getEntityId) + .map(Argument::getRefEntityId) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -69,24 +70,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { - Argument targetArgument = entry.getValue(); - String argumentKey = entry.getKey(); + ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); + String argumentName = entry.getKey(); - switch (targetArgument.getType()) { + switch (refEntityKey.getType()) { case ATTRIBUTE -> { - switch (targetArgument.getScope()) { + switch (refEntityKey.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(refEntityKey.getKey(), argumentName); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(refEntityKey.getKey(), argumentName); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(refEntityKey.getKey(), argumentName); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(refEntityKey.getKey(), argumentName); } }); @@ -118,7 +119,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel ObjectNode argumentsNode = configNode.putObject("arguments"); arguments.forEach((key, argument) -> { ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); + EntityId referencedEntityId = argument.getRefEntityId(); if (referencedEntityId != null) { argumentNode.put("entityType", referencedEntityId.getEntityType().name()); argumentNode.put("entityId", referencedEntityId.getId().toString()); @@ -126,9 +127,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityType", entityType.name()); argumentNode.put("entityId", entityId.toString()); } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", String.valueOf(argument.getType())); - argumentNode.put("scope", String.valueOf(argument.getScope())); +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", String.valueOf(argument.getType())); +// argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); @@ -165,19 +166,19 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { String referencedEntityType = argumentNode.get("entityType").asText(); UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } - argument.setKey(argumentNode.get("key").asText()); +// argument.setRefEntityKey(argumentNode.get("key").asText()); JsonNode type = argumentNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - argument.setType(ArgumentType.valueOf(type.asText())); - } - JsonNode scope = argumentNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - argument.setScope(AttributeScope.valueOf(scope.asText())); - } +// if (type != null && !type.isNull() && !type.asText().equals("null")) { +// argument.setType(ArgumentType.valueOf(type.asText())); +// } +// JsonNode scope = argumentNode.get("scope"); +// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { +// argument.setScope(AttributeScope.valueOf(scope.asText())); +// } if (argumentNode.hasNonNull("defaultValue")) { argument.setDefaultValue(argumentNode.get("defaultValue").asText()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..b49495d959 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6aaaf05836..d1352a26b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -22,6 +22,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -95,6 +96,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); +// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { @@ -112,6 +114,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); +// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 9a0b9222f0..f38722c9c1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -887,7 +887,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a8f7a2383..a9a70d8bc8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -156,7 +156,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index d0ee833261..236b159ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -382,7 +382,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 5b060ae145..cca30e6fbc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1225,7 +1225,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); From 3908105ac5e4cba45100a2020beb61db1a1e34c2 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 13 Jan 2025 13:23:41 +0200 Subject: [PATCH 077/438] fix_bug: tbel, in the isDecimal method. The mantissa format includes numbers written in exponential form --- .../java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- .../org/thingsboard/script/api/tbel/TbUtilsTest.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index fae91d14ca..ab1aef979a 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -1299,7 +1299,7 @@ public class TbUtils { if (str == null || str.isEmpty()) { return -1; } - return str.matches("[+-]?\\d+(\\.\\d+)?") ? DEC_RADIX : -1; + return str.matches("[+-]?\\d+(\\.\\d+)?([eE][+-]?\\d+)?") ? DEC_RADIX : -1; } public static int isHexadecimal(String str) { diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 21ee8538a0..4ac387b258 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -442,8 +442,9 @@ public class TbUtilsTest { @Test public void parsDouble() { - String doubleValStr = "1729.1729"; - Assertions.assertEquals(java.util.Optional.of(doubleVal).get(), TbUtils.parseDouble(doubleValStr)); + String doubleValStr = "1.1428250947E8"; + Assertions.assertEquals(Double.parseDouble(doubleValStr), TbUtils.parseDouble(doubleValStr)); + doubleValStr = "1729.1729"; Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseHexToDouble(longValHex))); Assertions.assertEquals(0, Double.compare(doubleValRev, TbUtils.parseHexToDouble(longValHex, false))); Assertions.assertEquals(0, Double.compare(doubleVal, TbUtils.parseBigEndianHexToDouble(longValHex))); @@ -930,7 +931,13 @@ public class TbUtilsTest { @Test public void isDecimal_Test() { Assertions.assertEquals(10, TbUtils.isDecimal("4567039")); + Assertions.assertEquals(10, TbUtils.isDecimal("1.1428250947E8")); + Assertions.assertEquals(10, TbUtils.isDecimal("123.45")); + Assertions.assertEquals(10, TbUtils.isDecimal("-1.23E-4")); + Assertions.assertEquals(10, TbUtils.isDecimal("1E5")); Assertions.assertEquals(-1, TbUtils.isDecimal("C100110")); + Assertions.assertEquals(-1, TbUtils.isDecimal("abc")); + Assertions.assertEquals(-1, TbUtils.isDecimal(null)); } @Test From f0a36d500883a382e540e861b0a36be20e5e9fbf Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:03:50 +0200 Subject: [PATCH 078/438] fixed tests --- .../BaseCalculatedFieldConfiguration.java | 129 +----------------- .../CalculatedFieldConfiguration.java | 6 - .../ScriptCalculatedFieldConfiguration.java | 12 -- .../SimpleCalculatedFieldConfiguration.java | 12 -- .../dao/model/sql/CalculatedFieldEntity.java | 16 +-- ...efaultNativeCalculatedFieldRepository.java | 11 +- .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 7 +- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 10 files changed, 20 insertions(+), 194 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8cab602683..87cece0419 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -15,48 +15,25 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.stream.Collectors; @Data public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - @JsonIgnore - private final ObjectMapper mapper = new ObjectMapper(); - protected Map arguments; protected String expression; protected Output output; - public BaseCalculatedFieldConfiguration() { - } - - public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { -// BaseCalculatedFieldConfiguration calculatedFieldConfig = mapper.convertValue(config, BaseCalculatedFieldConfiguration.class); - BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); - this.arguments = calculatedFieldConfig.getArguments(); - this.expression = calculatedFieldConfig.getExpression(); - this.output = calculatedFieldConfig.getOutput(); - } - @Override public List getReferencedEntities() { return arguments.values().stream() @@ -70,7 +47,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId() != null && entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); String argumentName = entry.getKey(); @@ -112,108 +89,4 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return link; } - @Override - public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { - ObjectNode configNode = mapper.createObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - arguments.forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getRefEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", String.valueOf(argument.getType())); -// argumentNode.put("scope", String.valueOf(argument.getScope())); - argumentNode.put("defaultValue", argument.getDefaultValue()); - argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); - }); - - if (expression != null) { - configNode.put("expression", expression); - } - - if (output != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("name", output.getName()); - outputNode.put("type", String.valueOf(output.getType())); - if (output.getScope() != null) { - outputNode.put("scope", String.valueOf(output.getScope())); - } - } - - return configNode; - } - - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null || !config.isObject()) { - return null; - } - - Map arguments = new HashMap<>(); - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - Argument argument = new Argument(); - if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } -// argument.setRefEntityKey(argumentNode.get("key").asText()); - JsonNode type = argumentNode.get("type"); -// if (type != null && !type.isNull() && !type.asText().equals("null")) { -// argument.setType(ArgumentType.valueOf(type.asText())); -// } -// JsonNode scope = argumentNode.get("scope"); -// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { -// argument.setScope(AttributeScope.valueOf(scope.asText())); -// } - if (argumentNode.hasNonNull("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); - } - if (argumentNode.hasNonNull("limit")) { - argument.setLimit(argumentNode.get("limit").asInt()); - } - if (argumentNode.hasNonNull("timeWindow")) { - argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); - } - arguments.put(key, argument); - }); - } - this.setArguments(arguments); - - JsonNode expressionNode = config.get("expression"); - if (expressionNode != null && expressionNode.isTextual()) { - this.setExpression(expressionNode.asText()); - } - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - Output output = new Output(); - output.setName(outputNode.get("name").asText()); - JsonNode type = outputNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - output.setType(OutputType.valueOf(type.asText())); - } - JsonNode scope = outputNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - output.setScope(AttributeScope.valueOf(scope.asText())); - } - this.setOutput(output); - } - - return this; - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index ac94ade134..8f56bf491d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -29,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; -import java.util.UUID; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -57,9 +54,6 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); - @JsonIgnore - JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); - List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java index a24328b4c9..017fc5a485 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public ScriptCalculatedFieldConfiguration() { - super(); - } - - public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index af11d2f5d8..6312c3e1db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public SimpleCalculatedFieldConfiguration() { - super(); - } - - public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index d1352a26b8..a0157cde66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,12 +23,9 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -96,8 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); -// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -113,8 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); -// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -122,11 +117,4 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - return switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index a5a2743f26..bb88982e3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +88,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(type, configuration, entityType, entityId)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); @@ -135,11 +133,4 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - return switch (type) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index f38722c9c1..aed1621e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -885,9 +886,9 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedAsset.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a9a70d8bc8..6dd84714f8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -154,9 +155,9 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 236b159ddb..6e57279f38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -380,9 +381,9 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedCustomer.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index cca30e6fbc..959825e113 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1223,9 +1224,9 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(device.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From d9ccc8118cf316389e3c284e468b4236118462bb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:52:06 +0200 Subject: [PATCH 079/438] fixed calculated field controller test --- .../server/controller/CalculatedFieldControllerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 565321afbc..b1c7547251 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -141,9 +142,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From 0f34f131c991012f9c499e36475064f0c4a7b97c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 21 Jan 2025 14:30:27 +0200 Subject: [PATCH 080/438] Return TimeseriesSaveResult --- .../DefaultTelemetrySubscriptionService.java | 35 +++++----- .../telemetry/InternalTelemetryService.java | 3 +- .../dao/timeseries/TimeseriesService.java | 9 +-- .../common/data/kv/TimeseriesSaveResult.java | 26 +++++++ .../dao/timeseries/BaseTimeseriesService.java | 69 ++++++++----------- 5 files changed, 79 insertions(+), 63 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..e7dfbcdca9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -126,10 +127,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer boolean sysTenant = TenantId.SYS_TENANT_ID.equals(tenantId) || tenantId == null; if (sysTenant || request.isOnlyLatest() || apiUsageStateService.getApiUsageState(tenantId).isDbStorageEnabled()) { KvUtils.validate(request.getEntries(), valueNoXssValidation); - ListenableFuture future = saveTimeseriesInternal(request); + ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -137,27 +137,27 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } @Override - public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { + public ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - ListenableFuture saveFuture; + ListenableFuture resultFuture; if (request.isOnlyLatest()) { - saveFuture = Futures.transform(tsService.saveLatest(tenantId, entityId, request.getEntries()), result -> 0, MoreExecutors.directExecutor()); + resultFuture = tsService.saveLatest(tenantId, entityId, request.getEntries()); } else if (request.isSaveLatest()) { - saveFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.save(tenantId, entityId, request.getEntries(), request.getTtl()); } else { - saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); + resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(saveFuture, request.getCallback()); - addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); + addMainCallback(resultFuture, request.getCallback()); + addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); - return saveFuture; + addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + return resultFuture; } @Override @@ -329,19 +329,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override - public void onSuccess(Integer result) { - if (!sysTenant && result != null && result > 0) { - apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); + public void onSuccess(TimeseriesSaveResult result) { + Integer dataPoints = result.getDataPoints(); + if (!sysTenant && dataPoints != null && dataPoints > 0) { + apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, dataPoints); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); } }; } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java index 8e45b84a75..8a236bf002 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/InternalTelemetryService.java @@ -21,13 +21,14 @@ import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; import org.thingsboard.rule.engine.api.TimeseriesDeleteRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; /** * Created by ashvayka on 27.03.18. */ public interface InternalTelemetryService extends RuleEngineTelemetryService { - ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); + ListenableFuture saveTimeseriesInternal(TimeseriesSaveRequest request); void saveAttributesInternal(AttributesSaveRequest request); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index 49918f3823..542f77445b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; @@ -44,13 +45,13 @@ public interface TimeseriesService { ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); + ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntry, long ttl); - ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntry); + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries); ListenableFuture> remove(TenantId tenantId, EntityId entityId, List queries); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java new file mode 100644 index 0000000000..edec063be2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TimeseriesSaveResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import lombok.Data; + +import java.util.List; + +@Data(staticConstructor = "of") +public class TimeseriesSaveResult { + private final Integer dataPoints; + private final List versions; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 756b73d88b..6b4a3c917e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; import org.thingsboard.server.dao.entityview.EntityViewService; @@ -156,60 +157,48 @@ public class BaseTimeseriesService implements TimeseriesService { } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { validate(entityId); - List> futures = new ArrayList<>(INSERTS_PER_ENTRY); - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, 0L); - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + return doSave(tenantId, entityId, List.of(tsKvEntry), 0L, true, true); } @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, true); + public ListenableFuture save(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, true, true); } @Override - public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { - return doSave(tenantId, entityId, tsKvEntries, ttl, false); - } - - private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest) { - int inserts = saveLatest ? INSERTS_PER_ENTRY : INSERTS_PER_ENTRY_WITHOUT_LATEST; - List> futures = new ArrayList<>(tsKvEntries.size() * inserts); - for (TsKvEntry tsKvEntry : tsKvEntries) { - if (saveLatest) { - saveAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } else { - saveWithoutLatestAndRegisterFutures(tenantId, futures, entityId, tsKvEntry, ttl); - } - } - return Futures.transform(Futures.allAsList(futures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()); + public ListenableFuture saveWithoutLatest(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl) { + return doSave(tenantId, entityId, tsKvEntries, ttl, false, true); } @Override - public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { - List> futures = new ArrayList<>(tsKvEntries.size()); - for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); - } - return Futures.allAsList(futures); + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { + return doSave(tenantId, entityId, tsKvEntries, 0L, true, false); } - private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); - } - - private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - } - - private void doSaveAndRegisterFuturesFor(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, List tsKvEntries, long ttl, boolean saveLatest, boolean saveTs) { + if (saveTs && entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } - futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); - futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + List> tsFutures = saveTs ? new ArrayList<>(tsKvEntries.size() * INSERTS_PER_ENTRY_WITHOUT_LATEST) : null; + List> latestFutures = saveLatest ? new ArrayList<>(tsKvEntries.size()) : null; + for (TsKvEntry tsKvEntry : tsKvEntries) { + if (saveTs) { + tsFutures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey())); + tsFutures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); + } + if (saveLatest) { + latestFutures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + } + } + ListenableFuture dpsFuture = saveTs ? Futures.transform(Futures.allAsList(tsFutures), SUM_ALL_INTEGERS, MoreExecutors.directExecutor()) : Futures.immediateFuture(0); + ListenableFuture> versionsFuture = saveLatest ? Futures.allAsList(latestFutures) : Futures.immediateFuture(null); + return Futures.whenAllComplete(dpsFuture, versionsFuture).call(() -> { + Integer dataPoints = Futures.getUnchecked(dpsFuture); + List versions = Futures.getUnchecked(versionsFuture); + return TimeseriesSaveResult.of(dataPoints, versions); + }, MoreExecutors.directExecutor()); } private List updateQueriesForEntityView(EntityView entityView, List queries) { From 9ef68584c9aecb19659496210cf914a7eb5f81b5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 09:40:12 +0200 Subject: [PATCH 081/438] updated getMappedTelemetry method --- ...efaultCalculatedFieldExecutionService.java | 19 ++++++++++--------- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 15 ++------------- ...CalculatedFieldTelemetryUpdateRequest.java | 5 +---- ...alculatedFieldTimeSeriesUpdateRequest.java | 13 +++---------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7c17d54649..84a21ab315 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -271,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) + .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -375,9 +375,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } @@ -406,9 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); @@ -427,13 +428,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } catch (Exception e) { @@ -654,7 +655,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) + EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e17a1a61f9..cb4052b7df 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -57,7 +57,7 @@ public class CalculatedFieldCtx { this.arguments = configuration.getArguments(); this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( - entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), Map.Entry::getKey )); this.argNames = new ArrayList<>(arguments.keySet()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 6050370fd2..d2eb31cd6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -52,16 +51,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return switch (scope) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); - }; - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -70,7 +60,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + String argName = referencedKeys.get(new TbPair<>(referencedEntityId, referencedEntityKey)); if (argName != null) { mappedKvEntries.put(argName, entry); @@ -79,5 +69,4 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel return mappedKvEntries; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index f85117dc41..3f7250f4ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,8 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); - - Map getMappedTelemetry(CalculatedFieldCtx ctx); + Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 646145a46e..a5637c8cfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -49,12 +48,7 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return linkConfiguration.getTimeSeries(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -62,13 +56,13 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe String key = entry.getKey(); ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + String argTsLatestName = referencedKeys.get(new TbPair<>(referencedEntityId, tsLatestKey)); if (argTsLatestName != null) { mappedKvEntries.put(argTsLatestName, entry); } else { ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + String argTsRollingName = referencedKeys.get(new TbPair<>(referencedEntityId, tsRollingKey)); if (argTsRollingName != null) { mappedKvEntries.put(argTsRollingName, entry); @@ -78,5 +72,4 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return mappedKvEntries; } - } From 6b9d374a5f2957d14a3722c2e6a6459da211db11 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 22 Jan 2025 12:23:42 +0200 Subject: [PATCH 082/438] Tmp commit for merge --- .../cf/CalculatedFieldExecutionService.java | 11 +++ .../TbCalculatedFieldConsumerService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 18 +++-- .../src/main/resources/thingsboard.yml | 12 ++- .../server/common/data/DataConstants.java | 2 + .../server/common/msg/queue/ServiceType.java | 3 +- common/proto/src/main/proto/queue.proto | 73 +++++++------------ .../queue/discovery/HashPartitionService.java | 12 ++- .../discovery/event/PartitionChangeEvent.java | 4 + 9 files changed, 86 insertions(+), 57 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 8ba1f6dfed..e18c8b4119 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -21,6 +21,17 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { + /** + * Push incoming telemetry to the CF processing queue for async processing. + * @param request - telemetry request; + * @param callback - callback to be executed when the message is ack by the queue. + */ + void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + + void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + + /* ===================================================== */ + void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..387bdd7143 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -0,0 +1,8 @@ +package org.thingsboard.server.service.queue; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface TbCalculatedFieldConsumerService extends ApplicationListener { + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 8773564e5d..dcb72b8dd0 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.telemetry; +import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -128,8 +129,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer KvUtils.validate(request.getEntries(), valueNoXssValidation); ListenableFuture future = saveTimeseriesInternal(request); if (!request.isOnlyLatest()) { - FutureCallback callback = getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant, request.getCallback()); - Futures.addCallback(future, callback, tsCallBackExecutor); + Futures.addCallback(future, getApiUsageCallback(tenantId, request.getCustomerId(), sysTenant), tsCallBackExecutor); } } else { request.getCallback().onFailure(new RuntimeException("DB storage writes are disabled due to API limits!")); @@ -148,7 +148,14 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - + // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. + saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { + @Override + public ListenableFuture apply(Integer input) throws Exception { + calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); + return input; + } + }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -326,19 +333,18 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } } - private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant, FutureCallback callback) { + private FutureCallback getApiUsageCallback(TenantId tenantId, CustomerId customerId, boolean sysTenant) { return new FutureCallback<>() { @Override public void onSuccess(Integer result) { if (!sysTenant && result != null && result > 0) { apiUsageClient.report(tenantId, customerId, ApiUsageRecordKey.STORAGE_DP_COUNT, result); } - callback.onSuccess(null); } @Override public void onFailure(Throwable t) { - callback.onFailure(t); + } }; } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3014e1448b..5151bc019b 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1692,7 +1692,6 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" - vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" @@ -1739,6 +1738,17 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + calculated-fields: + # Topic name for Calculated Field (CF) tasks + topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + # Interval in milliseconds to poll messages by CF (Rule Engine) microservices + poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + # Amount of partitions used by CF microservices + partitions: "${TB_QUEUE_CF_PARTITIONS:10}" + # Timeout for processing a message pack by CF microservices + pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + # Enable/disable a separate consumer per partition for CF queue + consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 56a5e135f8..77a7c4a781 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -145,4 +145,6 @@ public class DataConstants { public static final String EDGE_QUEUE_NAME = "Edge"; public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; + public static final String CF_QUEUE_NAME = "CalculatedFields"; + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f3a0e47d09..022c46bcc6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + TB_CF_ENGINE("TB Calculated Fields Engine"); private final String label; diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 1036d5ba67..4e719a02df 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,18 +183,6 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } -message AttributeKvProto { - AttributeKey key = 1; - AttributeValueProto value = 2; -} - -message TelemetryProto { - oneof proto { - AttributeKvProto attrKv = 1; - TsKvProto tsKv = 2; - } -} - message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -785,17 +773,7 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - bool added = 5; - bool updated = 6; - bool deleted = 7; -} - -message EntityProfileUpdateMsgProto { +message CalculatedFieldEntityUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; @@ -806,31 +784,26 @@ message EntityProfileUpdateMsgProto { int64 oldProfileIdLSB = 8; int64 newProfileIdMSB = 9; int64 newProfileIdLSB = 10; + bool added = 11; + bool updated = 12; + bool deleted = 13; } -message ProfileEntityMsgProto { +message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; string entityType = 3; int64 entityIdMSB = 4; int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 profileIdMSB = 7; - int64 profileIdLSB = 8; - bool added = 9; - bool deleted = 10; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + repeated TsKvProto tsData = 9; + AttributeScopeProto scope = 10; + repeated AttributeValueProto attrData = 11; } -message TelemetryUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - repeated CalculatedFieldEntityCtxIdProto links = 6; - repeated CalculatedFieldIdProto previousCalculatedFields = 7; - string scope = 8; - repeated TelemetryProto updatedTelemetry = 9; +message CalculatedFieldLinkedTelemetryMsgProto { + CalculatedFieldTelemetryMsgProto msg = 1; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1589,9 +1562,8 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; - CalculatedFieldMsgProto calculatedFieldMsg = 53; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; - ProfileEntityMsgProto profileEntityMsg = 55; +// CalculatedFieldMsgProto calculatedFieldMsg = 53; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1611,8 +1583,8 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; - ProfileEntityMsgProto profileEntityMsg = 52; +// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; +// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ @@ -1632,6 +1604,16 @@ message ToEdgeEventNotificationMsg { EdgeEventMsgProto edgeEventMsg = 1; } +message ToCalculatedFieldMsg { + CalculatedFieldTelemetryMsgProto telemetryMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; +} + +message ToCalculatedFieldNotificationMsg { + ComponentLifecycleMsgProto componentLifecycle = 1; + CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; +} + /* Messages that are handled by ThingsBoard RuleEngine Service */ message ToRuleEngineMsg { int64 tenantIdMSB = 1; @@ -1639,9 +1621,6 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; - TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; - EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; - ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 37e519e3f2..53bdc78c93 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,8 +51,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; -import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.*; @Service @Slf4j @@ -62,6 +61,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; + @Value("${queue.calculated-fields.topic}") + private String cfTopic; + @Value("${queue.calculated-fields.partitions:10}") + private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") @@ -108,10 +111,15 @@ public class HashPartitionService implements PartitionService { @PostConstruct public void init() { this.hashFunction = forName(hashFunctionName); + QueueKey coreKey = new QueueKey(ServiceType.TB_CORE); partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + partitionSizesMap.put(cfKey, cfPartitions); + partitionTopicsMap.put(cfKey, cfTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); partitionTopicsMap.put(vcKey, vcTopic); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 88ceb4aa08..3bb0c56f9a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -52,6 +52,10 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getCalculatedFieldsPartitions() { + return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { return partitionsMap.entrySet() .stream() From c047d5f4f065d17ea80935bb4f3ebb9eae518c80 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 17:01:28 +0200 Subject: [PATCH 083/438] added debug event entity --- .../server/actors/ActorSystemContext.java | 41 ++++++ .../actors/ruleChain/DefaultTbContext.java | 2 +- ...efaultCalculatedFieldExecutionService.java | 13 +- .../common/data/cf/CalculatedField.java | 23 ++- .../data/event/CalculatedFieldDebugEvent.java | 95 ++++++++++++ .../CalculatedFieldDebugEventFilter.java | 50 +++++++ .../server/common/data/event/EventType.java | 3 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + .../server/dao/event/BaseEventService.java | 6 + .../server/dao/model/ModelConstants.java | 5 + .../sql/CalculatedFieldDebugEventEntity.java | 104 ++++++++++++++ .../dao/model/sql/CalculatedFieldEntity.java | 7 + .../CalculatedFieldDebugEventRepository.java | 135 ++++++++++++++++++ .../dao/sql/event/DedicatedJpaEventDao.java | 5 +- .../dao/sql/event/EventInsertRepository.java | 29 ++++ .../server/dao/sql/event/JpaBaseEventDao.java | 47 ++++++ .../main/resources/sql/schema-entities.sql | 17 +++ 17 files changed, 570 insertions(+), 13 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 71fa8fa958..e6f541090c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -42,10 +42,12 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -125,6 +127,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -157,6 +160,18 @@ public class ActorSystemContext { } }; + private static final FutureCallback CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Calculated Field", th); + } + }; + private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); public ConcurrentMap getDebugPerTenantLimits() { @@ -723,6 +738,32 @@ public class ActorSystemContext { } } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { + if (checkLimits(tenantId, tbMsg, error)) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(tbMsg.getOriginator()) + .msgId(tbMsg.getId()) + .msgType(tbMsg.getType()) + .arguments(JacksonUtil.toString(arguments)) + .result(tbMsg.getData()); + + if (error != null) { + event.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + } + } + private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java index 421e4efb26..54e32446bf 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.netty.channel.EventLoopGroup; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.MailService; @@ -64,7 +65,6 @@ import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.common.data.script.ScriptLanguage; import org.thingsboard.server.common.msg.TbActorMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..a365e2bb6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -71,6 +70,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -122,6 +122,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; + private final EventService eventService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -253,7 +254,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; @@ -301,6 +301,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { + calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); callback.onSuccess(); shouldReinit = false; } @@ -329,13 +330,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration(); - CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration(); - boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments()); - boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType()); - boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression()); + boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged; + return entityIdChanged || typeChanged || argumentsChanged; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index e626c9d3d2..f4b92b3802 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.cf; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; @@ -22,11 +24,13 @@ import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.HasVersion; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -36,7 +40,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity { +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, ExportableEntity, HasDebugSettings { private static final long serialVersionUID = 4491966747773381420L; @@ -50,6 +54,11 @@ public class CalculatedField extends BaseData implements HasN @Length(fieldName = "name") @Schema(description = "User defined name of the calculated field.") private String name; + @Deprecated + @Schema(description = "Enable/disable debug. ", example = "false", deprecated = true) + private boolean debugMode; + @Schema(description = "Debug settings object.") + private DebugSettings debugSettings; @Schema(description = "Version of calculated field configuration.", example = "0") private int configurationVersion; @Schema(implementation = SimpleCalculatedFieldConfiguration.class) @@ -109,4 +118,16 @@ public class CalculatedField extends BaseData implements HasN .toString(); } + // Getter is ignored for serialization + @JsonIgnore + public boolean isDebugMode() { + return debugMode; + } + + // Setter is annotated for deserialization + @JsonSetter + public void setDebugMode(boolean debugMode) { + this.debugMode = debugMode; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java new file mode 100644 index 0000000000..e0599db358 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -0,0 +1,95 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class CalculatedFieldDebugEvent extends Event { + + private static final long serialVersionUID = -7091690784759639853L; + + @Builder + private CalculatedFieldDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + CalculatedFieldId calculatedFieldId, EntityId eventEntity, UUID msgId, + String msgType, String arguments, String result, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.calculatedFieldId = calculatedFieldId; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.arguments = arguments; + this.result = result; + this.error = error; + } + + @Getter + private final CalculatedFieldId calculatedFieldId; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + @Setter + private String arguments; + @Getter + @Setter + private String result; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("calculatedFieldId", calculatedFieldId.toString()); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "arguments", arguments); + putNotNull(json, "result", result); + putNotNull(json, "error", error); + return eventInfo; + } + + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java new file mode 100644 index 0000000000..839583ef6c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEventFilter.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; + +@Data +@EqualsAndHashCode(callSuper = true) +@Schema +public class CalculatedFieldDebugEventFilter extends DebugEventFilter { + + @Schema(description = "String value representing the calculated field id in the event body", example = "ccbfa2fe-c8f5-45d8-bb37-6b61a6e02833") + protected String calculatedFieldId; + @Schema(description = "String value representing the entity id in the event body", example = "57b6bafe-d600-423c-9267-fe31e5218986") + protected String entityId; + @Schema(description = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @Schema(description = "String value representing the message id in the rule engine", example = "dcf44612-2ce4-4e5d-b462-ebb9c5628228") + protected String msgId; + @Schema(description = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + protected String msgType; + + @Override + public EventType getEventType() { + return EventType.DEBUG_CALCULATED_FIELD; + } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(calculatedFieldId) || !StringUtils.isEmpty(entityId) + || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 6f98e1537f..46d0e49370 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -22,7 +22,8 @@ public enum EventType { LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), - DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true), + DEBUG_CALCULATED_FIELD("cf_debug_event", "DEBUG_CALCULATED_FIELD", true); @Getter private final String table; diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 9c81d91f64..26ed4134cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -64,6 +64,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements log.trace("Executing save calculated field, [{}]", calculatedField); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 5314dcd405..dc57b07f72 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -88,6 +89,11 @@ public class BaseEventService implements EventService { ErrorEvent eEvent = (ErrorEvent) event; truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); break; + case DEBUG_CALCULATED_FIELD: + CalculatedFieldDebugEvent cfEvent = (CalculatedFieldDebugEvent) event; + truncateField(cfEvent, CalculatedFieldDebugEvent::getArguments, CalculatedFieldDebugEvent::setArguments); + truncateField(cfEvent, CalculatedFieldDebugEvent::getResult, CalculatedFieldDebugEvent::setResult); + truncateField(cfEvent, CalculatedFieldDebugEvent::getError, CalculatedFieldDebugEvent::setError); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 6bd6a3a384..0cc2f03d2e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -373,6 +373,7 @@ public class ModelConstants { public static final String STATS_EVENT_TABLE_NAME = "stats_event"; public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME = "cf_debug_event"; public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; @@ -397,6 +398,10 @@ public class ModelConstants { public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String EVENT_CALCULATED_FIELD_ID_COLUMN_NAME = "cf_id"; + public static final String EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME = "e_args"; + public static final String EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME = "e_result"; + public static final String DEBUG_MODE = "debug_mode"; public static final String DEBUG_SETTINGS = "debug_settings"; public static final String SINGLETON_MODE = "singleton_mode"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java new file mode 100644 index 0000000000..57849ae3ea --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldDebugEventEntity.java @@ -0,0 +1,104 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Table; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = CALCULATED_FIELD_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class CalculatedFieldDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_CALCULATED_FIELD_ID_COLUMN_NAME) + private UUID calculatedFieldId; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_CALCULATED_FIELD_ARGUMENTS_COLUMN_NAME) + private String arguments; + @Column(name = EVENT_CALCULATED_FIELD_RESULT_COLUMN_NAME) + private String result; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public CalculatedFieldDebugEventEntity(CalculatedFieldDebugEvent event) { + super(event); + if (event.getCalculatedFieldId() != null) { + this.calculatedFieldId = event.getCalculatedFieldId().getId(); + } + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.arguments = event.getArguments(); + this.result = event.getResult(); + this.error = event.getError(); + } + + @Override + public CalculatedFieldDebugEvent toData() { + var builder = CalculatedFieldDebugEvent.builder() + .id(id) + .tenantId(TenantId.fromUUID(tenantId)) + .ts(ts) + .serviceId(serviceId) + .entityId(entityId) + .msgId(msgId) + .msgType(msgType) + .arguments(arguments) + .result(result) + .error(error); + if (calculatedFieldId != null) { + builder.calculatedFieldId(new CalculatedFieldId(calculatedFieldId)); + } + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index a0157cde66..64c8e8d5b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -26,6 +26,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -45,6 +46,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_T import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TENANT_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_TYPE; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_VERSION; +import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @Data @EqualsAndHashCode(callSuper = true) @@ -77,6 +79,9 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem @Column(name = CALCULATED_FIELD_VERSION) private Long version; + @Column(name = DEBUG_SETTINGS) + private String debugSettings; + @Column(name = CALCULATED_FIELD_EXTERNAL_ID) private UUID externalId; @@ -95,6 +100,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.configurationVersion = calculatedField.getConfigurationVersion(); this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); + this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); } @@ -111,6 +117,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java new file mode 100644 index 0000000000..c0bd21ca74 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -0,0 +1,135 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; +import org.thingsboard.server.dao.model.sql.CalculatedFieldDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface CalculatedFieldDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM cf_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM CalculatedFieldDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM cf_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:calculatedFieldId IS NULL OR e.cf_id = uuid(:calculatedFieldId)) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("calculatedFieldId") UUID calculatedFieldId, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("isError") boolean isError, + @Param("error") String error); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java index 9b7af5e7f7..7bf9b426e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/DedicatedJpaEventDao.java @@ -36,10 +36,11 @@ public class DedicatedJpaEventDao extends JpaBaseEventDao { RuleNodeDebugEventRepository ruleNodeDebugEventRepository, RuleChainDebugEventRepository ruleChainDebugEventRepository, ScheduledLogExecutorComponent logExecutor, - StatsFactory statsFactory) { + StatsFactory statsFactory, + CalculatedFieldDebugEventRepository cfDebugEventRepository) { super(partitionConfiguration, partitioningRepository, lcEventRepository, statsEventRepository, errorEventRepository, eventInsertRepository, ruleNodeDebugEventRepository, - ruleChainDebugEventRepository, logExecutor, statsFactory); + ruleChainDebugEventRepository, logExecutor, statsFactory, cfDebugEventRepository); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 962be57892..9307b9e9be 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -25,6 +25,7 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -81,6 +82,9 @@ public class EventInsertRepository { insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + + " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } public void save(List entities) { @@ -107,6 +111,8 @@ public class EventInsertRepository { return getRuleNodeEventSetter(events); case DEBUG_RULE_CHAIN: return getRuleChainEventSetter(events); + case DEBUG_CALCULATED_FIELD: + return getCalculatedFieldEventSetter(events); default: throw new RuntimeException(eventType + " support is not implemented!"); } @@ -206,6 +212,29 @@ public class EventInsertRepository { }; } + private BatchPreparedStatementSetter getCalculatedFieldEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + CalculatedFieldDebugEvent event = (CalculatedFieldDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutUUID(ps, 6, event.getCalculatedFieldId().getId()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getArguments()); + safePutString(ps, 12, event.getResult()); + safePutString(ps, 13, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { if (value != null) { ps.setString(parameterIdx, replaceNullChars(value)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index b8d5083402..e3c1b37536 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.CalculatedFieldDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; @@ -72,6 +73,7 @@ public class JpaBaseEventDao implements EventDao { private final RuleChainDebugEventRepository ruleChainDebugEventRepository; private final ScheduledLogExecutorComponent logExecutor; private final StatsFactory statsFactory; + private final CalculatedFieldDebugEventRepository calculatedFieldDebugEventRepository; @Value("${sql.events.batch_size:10000}") private int batchSize; @@ -110,6 +112,7 @@ public class JpaBaseEventDao implements EventDao { repositories.put(EventType.ERROR, errorEventRepository); repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); + repositories.put(EventType.DEBUG_CALCULATED_FIELD, calculatedFieldDebugEventRepository); } @PreDestroy @@ -158,6 +161,8 @@ public class JpaBaseEventDao implements EventDao { return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); case STATS: return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + case DEBUG_CALCULATED_FIELD: + return findEventByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, pageLink); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -193,6 +198,8 @@ public class JpaBaseEventDao implements EventDao { case STATS: removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); break; + case DEBUG_CALCULATED_FIELD: + removeEventsByFilter(tenantId, entityId, (CalculatedFieldDebugEventFilter) eventFilter, startTime, endTime); default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } @@ -286,6 +293,27 @@ public class JpaBaseEventDao implements EventDao { ); } + private PageData findEventByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + calculatedFieldDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink, EventEntity.eventColumnMap))); + } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { ruleChainDebugEventRepository.removeEvents( tenantId, @@ -360,6 +388,25 @@ public class JpaBaseEventDao implements EventDao { ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, CalculatedFieldDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getCalculatedFieldId(), "Calculated Field Id"); + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + calculatedFieldDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + UUID.fromString(eventFilter.getCalculatedFieldId()), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 51cc66c4ac..a6d1e8800d 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -934,6 +934,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration_version int DEFAULT 0, configuration varchar(1000000), version BIGINT DEFAULT 1, + debug_settings varchar(1024), external_id UUID, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, name), CONSTRAINT calculated_field_external_id_unq_key UNIQUE (tenant_id, external_id) @@ -949,3 +950,19 @@ CREATE TABLE IF NOT EXISTS calculated_field_link ( configuration varchar(10000), CONSTRAINT fk_calculated_field_id FOREIGN KEY (calculated_field_id) REFERENCES calculated_field(id) ON DELETE CASCADE ); + +CREATE TABLE IF NOT EXISTS cf_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar, + cf_id uuid NOT NULL, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_args varchar, + e_result varchar, + e_error varchar +) PARTITION BY RANGE (ts); From 069725a2b95dd22f8e7ab59250904c8d8897d11f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 10:43:26 +0200 Subject: [PATCH 084/438] WIP Refactoring of the cluster mode --- ...faultTbCalculatedFieldConsumerService.java | 245 ++++++++++++++++++ .../DefaultTbRuleEngineConsumerService.java | 5 + .../DefaultTelemetrySubscriptionService.java | 14 +- .../src/main/resources/thingsboard.yml | 16 +- .../server/common/util/ProtoUtils.java | 30 +-- common/proto/src/main/proto/queue.proto | 29 +++ .../queue/discovery/HashPartitionService.java | 15 +- .../server/queue/discovery/TopicService.java | 12 +- .../discovery/event/PartitionChangeEvent.java | 5 +- .../InMemoryMonolithQueueFactory.java | 34 +++ .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbRuleEngineQueueFactory.java | 24 +- .../TbQueueCalculatedFieldSettings.java | 35 +++ 13 files changed, 430 insertions(+), 45 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java new file mode 100644 index 0000000000..85ee01ac5f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -0,0 +1,245 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; +import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; + +import java.util.List; +import java.util.UUID; + +@Service +@TbRuleEngineComponent +@Slf4j +public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { + + @Value("${queue.calculated_fields.poll_interval}") + private long pollInterval; + @Value("${queue.calculated_fields.pack_processing_timeout}") + private long packProcessingTimeout; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + @Value("${queue.calculated_fields.pool_size:8}") + private int poolSize; + + private final TbRuleEngineQueueFactory queueFactory; + + private final CalculatedFieldExecutionService calculatedFieldExecutionService; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; + + private volatile ListeningExecutorService calculatedFieldsExecutor; + + public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, + ActorSystemContext actorContext, + TbDeviceProfileCache deviceProfileCache, + TbAssetProfileCache assetProfileCache, + TbTenantProfileCache tenantProfileCache, + TbApiUsageStateService apiUsageStateService, + PartitionService partitionService, + ApplicationEventPublisher eventPublisher, + JwtSettingsService jwtSettingsService, + CalculatedFieldExecutionService calculatedFieldExecutionService, + CalculatedFieldCache calculatedFieldCache) { + super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, + eventPublisher, jwtSettingsService); + this.queueFactory = tbQueueFactory; + this.calculatedFieldExecutionService = calculatedFieldExecutionService; + } + + @PostConstruct + public void init() { + super.init("tb-cf"); + this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + + this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(new QueueKey(ServiceType.TB_CORE)) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor(this::processMsgs) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + } + + @PreDestroy + public void destroy() { + super.destroy(); + if (calculatedFieldsExecutor != null) { + calculatedFieldsExecutor.shutdownNow(); + } + } + + @Override + protected void startConsumers() { + super.startConsumers(); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + mainConsumer.update(event.getCalculatedFieldsPartitions()); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { + + } + + @Override + protected ServiceType getServiceType() { + return ServiceType.TB_RULE_ENGINE; + } + + @Override + protected long getNotificationPollDuration() { + return pollInterval; + } + + @Override + protected long getNotificationPackProcessingTimeout() { + return packProcessingTimeout; + } + + @Override + protected int getMgmtThreadPoolSize() { + return Math.max(Runtime.getRuntime().availableProcessors(), 4); + } + + @Override + protected TbQueueConsumer> createNotificationsConsumer() { + return queueFactory.createToCalculatedFieldNotificationsMsgConsumer(); + } + + @Override + protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { + ToCalculatedFieldNotificationMsg notification = msg.getValue(); + + callback.onSuccess(); + } + +// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); +// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); +// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); +// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); +// } +// +// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { +// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); +// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); +// boolean added = profileEntityMsg.getAdded(); +// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); +// if (added) { +// entitiesByProfile.add(entityId); +// } else { +// entitiesByProfile.remove(entityId); +// } +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { +// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); +// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { +// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } +// +// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { +// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); +// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); +// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); +// DonAsynchron.withCallback(future, +// __ -> callback.onSuccess(), +// t -> { +// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); +// callback.onFailure(t); +// }); +// } + + private void throwNotHandled(Object msg, TbCallback callback) { + log.warn("Message not handled: {}", msg); + callback.onFailure(new RuntimeException("Message not handled!")); + } + + private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { + return TenantId.fromUUID(new UUID(tenantIdMSB, tenantIdLSB)); + } + + @Override + protected void stopConsumers() { + super.stopConsumers(); + mainConsumer.stop(); + mainConsumer.awaitStop(); + } + + @Data(staticConstructor = "of") + public static class CalculatedFieldQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 7d4d975cb4..5e07e33a9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -63,6 +63,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @Service @TbRuleEngineComponent @Slf4j @@ -107,6 +109,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { + if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + return; + } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index dcb72b8dd0..fdf3ed6c50 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -149,13 +149,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer saveFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } // We need to guarantee, that the message is successfully pushed to the calculated fields service before we execute any callbacks. - saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { - @Override - public ListenableFuture apply(Integer input) throws Exception { - calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); - return input; - } - }); +// saveFuture = Futures.transformAsync(saveFuture, new AsyncFunction() { +// @Override +// public ListenableFuture apply(Integer input) throws Exception { +// calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)); +// return input; +// } +// }); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5151bc019b..94c1bef71c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1738,17 +1738,21 @@ queue: topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" - calculated-fields: - # Topic name for Calculated Field (CF) tasks - topic: "${TB_QUEUE_CF_TOPIC:tb_calculated_fields}" + calculated_fields: + # Topic name for Calculated Field (CF) events from Rule Engine + event_topic: "${TB_QUEUE_CF_EVENT_TOPIC:tb_cf_event}" + # Topic name for Calculated Field (CF) compacted states + state_topic: "${TB_QUEUE_CF_STATE_TOPIC:tb_cf_state}" # Interval in milliseconds to poll messages by CF (Rule Engine) microservices - poll-interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" + poll_interval: "${TB_QUEUE_CF_POLL_INTERVAL_MS:25}" # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack-processing-timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" # Enable/disable a separate consumer per partition for CF queue - consumer-per-partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" + # Thread pool size for processing of the incoming messages + pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 073f47d59b..1b743316bd 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,15 +644,15 @@ public class ProtoUtils { return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { - if (telemetryProto.hasAttrKv()) { - return fromProto(telemetryProto.getAttrKv().getValue()); - } else if (telemetryProto.hasTsKv()) { - return fromProto(telemetryProto.getTsKv()); - } else { - throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); - } - } +// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { +// if (telemetryProto.hasAttrKv()) { +// return fromProto(telemetryProto.getAttrKv().getValue()); +// } else if (telemetryProto.hasTsKv()) { +// return fromProto(telemetryProto.getTsKv()); +// } else { +// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); +// } +// } public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); @@ -673,12 +673,12 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { - return TransportProtos.AttributeKvProto.newBuilder() - .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) - .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) - .build(); - } +// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { +// return TransportProtos.AttributeKvProto.newBuilder() +// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) +// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) +// .build(); +// } public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4e719a02df..288c923aaa 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -819,6 +819,35 @@ message CalculatedFieldIdProto { int64 calculatedFieldIdLSB = 2; } +message SingleValueProto { + int64 ts = 1; + int64 version = 2; + KeyValueType type = 3; + bool has_v = 4; + bool bool_v = 5; + int64 long_v = 6; + double double_v = 7; + string string_v = 8; + string json_v = 9; +} + +message SingleValueArgumentProto { + string argName = 1; + SingleValueProto value = 2; +} + +message RollingArgumentProto { + string argName = 1; + repeated SingleValueProto values = 2; +} + +message CalculatedFieldStateProto { + CalculatedFieldEntityCtxIdProto id = 1; + // int32 version = 2; + repeated SingleValueArgumentProto singleValueArguments = 3; + repeated RollingArgumentProto rollingValueArguments = 4; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 53bdc78c93..7ac938f52f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -61,9 +61,11 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:10}") private Integer corePartitions; - @Value("${queue.calculated-fields.topic}") - private String cfTopic; - @Value("${queue.calculated-fields.partitions:10}") + @Value("${queue.calculated_fields.event_topic}") + private String cfEventTopic; + @Value("${queue.calculated_fields.state_topic}") + private String cfStateTopic; + @Value("${queue.calculated_fields.partitions:10}") private Integer cfPartitions; @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @@ -76,6 +78,8 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; + public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -116,9 +120,8 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - QueueKey cfKey = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - partitionSizesMap.put(cfKey, cfPartitions); - partitionTopicsMap.put(cfKey, cfTopic); + partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); + partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 927c311a2d..8dec36c15c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -35,6 +35,7 @@ public class TopicService { private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); + private final ConcurrentMap tbCalculatedFieldNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentReferenceHashMap tbEdgeEventsNotificationTopics = new ConcurrentReferenceHashMap<>(); /** @@ -62,6 +63,11 @@ public class TopicService { return buildTopicPartitionInfo("tb_edge.notifications." + serviceId, null, null, false); } + public TopicPartitionInfo getCalculatedFieldNotificationsTopic(String serviceId) { + return tbCalculatedFieldNotificationTopics.computeIfAbsent(serviceId, + id -> buildNotificationsTopicPartitionInfo("calculated_field", serviceId)); + } + public TopicPartitionInfo getEdgeEventNotificationsTopic(TenantId tenantId, EdgeId edgeId) { return tbEdgeEventsNotificationTopics.computeIfAbsent(edgeId, id -> buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId)); } @@ -71,7 +77,11 @@ public class TopicService { } private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) { - return buildTopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false); + return buildNotificationsTopicPartitionInfo(serviceType.name().toLowerCase(), serviceId); + } + + private TopicPartitionInfo buildNotificationsTopicPartitionInfo(String serviceType, String serviceId) { + return buildTopicPartitionInfo(serviceType + ".notifications." + serviceId, null, null, false); } public TopicPartitionInfo buildTopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 3bb0c56f9a..57a4941981 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,10 +23,13 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; + @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -53,7 +56,7 @@ public class PartitionChangeEvent extends TbApplicationEvent { } public Set getCalculatedFieldsPartitions() { - return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index d70cad159b..c26e2d15c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -33,6 +33,7 @@ import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -53,6 +54,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, @@ -62,6 +64,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, InMemoryStorage storage) { this.topicService = topicService; this.coreSettings = coreSettings; @@ -71,6 +74,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.storage = storage; } @@ -139,6 +143,31 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); @@ -209,6 +238,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index c4002f4d3e..0b3df5bccf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -159,4 +160,6 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index c406aeb311..76dad05393 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,9 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -109,11 +112,22 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory } /** - * Used to consume high priority messages by TB Core Service + * Used to consume high priority messages by TB Rule Engine Service * * @return */ TbQueueConsumer> createToRuleEngineNotificationsMsgConsumer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + + TbQueueProducer> createToCalculatedFieldMsgProducer(); + + TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); + + TbQueueConsumer> createCalculatedFieldStateConsumer(); + + TbQueueProducer> createCalculatedFieldStateProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java new file mode 100644 index 0000000000..22bbd7e0f3 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueCalculatedFieldSettings { + + @Value("${queue.calculated_fields.event_topic}") + private String eventTopic; + + @Value("${queue.calculated_fields.state_topic}") + private String stateTopic; + + +} From 4c71b9d5f68907dc229fa181a3c3f43aafcf84b7 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jan 2025 13:56:44 +0200 Subject: [PATCH 085/438] WIP: Cluster mode implementation --- .../service/cf/CalculatedFieldCache.java | 4 +- .../cf/CalculatedFieldExecutionService.java | 12 +- .../cf/DefaultCalculatedFieldCache.java | 11 +- ...efaultCalculatedFieldExecutionService.java | 115 +++++++++++++++++- .../cf/ctx/state/CalculatedFieldCtx.java | 43 +++++++ ...faultTbCalculatedFieldConsumerService.java | 8 +- .../queue/DefaultTbClusterService.java | 12 +- .../TbCalculatedFieldConsumerService.java | 15 +++ .../AbstractSubscriptionService.java | 8 ++ .../DefaultTelemetrySubscriptionService.java | 38 +++--- .../server/cluster/TbClusterService.java | 3 + .../queue/discovery/HashPartitionService.java | 3 +- .../queue/discovery/PartitionService.java | 2 + .../queue/provider/TbCoreQueueFactory.java | 11 +- .../provider/TbCoreQueueProducerProvider.java | 16 +++ .../provider/TbQueueProducerProvider.java | 8 +- .../TbRuleEngineProducerProvider.java | 14 +++ .../provider/TbRuleEngineQueueFactory.java | 8 +- .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 11 ++ 20 files changed, 298 insertions(+), 54 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 8730aeeedf..ea55894432 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,9 +36,9 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e18c8b4119..2507546013 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -22,13 +24,13 @@ import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdat public interface CalculatedFieldExecutionService { /** - * Push incoming telemetry to the CF processing queue for async processing. - * @param request - telemetry request; - * @param callback - callback to be executed when the message is ack by the queue. + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * @param request - telemetry save request; + * @param request - telemetry save result; */ - void pushRequestToQueue(CalculatedFieldTelemetryUpdateRequest request, TbCallback callback); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); +// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 868001d0d2..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -55,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final CalculatedFieldService calculatedFieldService; private final AssetService assetService; private final DeviceService deviceService; + private final TbelInvokeService tbelInvokeService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -105,7 +107,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); @@ -128,9 +130,12 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + public List getCalculatedFieldCtxsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } return getCalculatedFieldsByEntityId(entityId).stream() - .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> getCalculatedFieldCtx(cf.getId())) .toList(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 84a21ab315..0f9b948a9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,12 +33,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -60,6 +61,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; @@ -72,7 +74,9 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -103,6 +107,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; @@ -113,6 +119,16 @@ import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @RequiredArgsConstructor public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -121,7 +137,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; - private final TbelInvokeService tbelInvokeService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; @@ -169,6 +184,70 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return "calculated-field-scheduled"; } + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ////TODO: IM to push to CF queue + return null; + } + + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) + .forEach(link -> { + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); + }); + } else { + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); + } + }); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -374,7 +453,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; @@ -391,7 +470,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { @@ -830,4 +909,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index cb4052b7df..1a9f999497 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -20,15 +20,18 @@ import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -41,6 +44,9 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; + private final Map mainEntityArguments; + private final Map> linkedEntityArguments; + private final Map, String> referencedEntityKeys; private final List argNames; private Output output; @@ -55,6 +61,17 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); + this.mainEntityArguments = new HashMap<>(); + this.linkedEntityArguments = new HashMap<>(); + for (Map.Entry entry : arguments.entrySet()) { + var refId = entry.getValue().getRefEntityId(); + var refKey = entry.getValue().getRefEntityKey(); + if (refId == null) { + mainEntityArguments.put(refKey, entry.getKey()); + } else { + linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); + } + } this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), @@ -82,4 +99,30 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values) { + return matches(mainEntityArguments, values); + } + + public boolean linkMatches(EntityId entityId, List values) { + var map = linkedEntityArguments.get(entityId); + if (map == null) { + return false; + } else { + return matches(map, values); + } + } + + private static boolean matches(Map argMap, List values) { + for (TsKvEntry tsKv : values) { + ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); + if (argMap.containsKey(latestKey)) { + return true; + } + ReferencedEntityKey rollingKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_ROLLING, null); + if (argMap.containsKey(rollingKey)) { + return true; + } + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 551b0ddbc2..725e0fdd79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,7 +69,8 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -108,7 +109,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -336,6 +339,13 @@ public class DefaultTbClusterService implements TbClusterService { toTransportNfs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 0bb8f76398..21cb902ad6 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -116,4 +116,12 @@ public abstract class AbstractSubscriptionService extends TbApplicationEventList }, executor); } + protected static Consumer safeCallback(FutureCallback callback) { + if (callback != null) { + return callback::onFailure; + } else { + return throwable -> {}; + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index c9602b4650..71716a16e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.telemetry; -import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -53,7 +52,6 @@ import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -149,14 +147,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } else { resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } - addMainCallback(resultFuture, request.getCallback()); + DonAsynchron.withCallback(resultFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request, result); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(resultFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); return resultFuture; } @@ -171,6 +168,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); + //TODO: IM to push to CF queue addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @@ -268,27 +266,21 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer } private void onAttributesUpdate(TenantId tenantId, EntityId entityId, String scope, List attributes, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesUpdate(tenantId, entityId, scope, attributes, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesUpdateProto(tenantId, entityId, scope, attributes)); } private void onAttributesDelete(TenantId tenantId, EntityId entityId, String scope, List keys, boolean notifyDevice) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onAttributesDelete(tenantId, entityId, scope, keys, notifyDevice, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toAttributesDeleteProto(tenantId, entityId, scope, keys, notifyDevice)); } private void onTimeSeriesUpdate(TenantId tenantId, EntityId entityId, List ts) { - forwardToSubscriptionManagerService(tenantId, entityId, subscriptionManagerService -> { - subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts); - }); + forwardToSubscriptionManagerService(tenantId, entityId, + subscriptionManagerService -> subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, ts, TbCallback.EMPTY), + () -> TbSubscriptionUtils.toTimeseriesUpdateProto(tenantId, entityId, ts)); } private void onTimeSeriesDelete(TenantId tenantId, EntityId entityId, List keys, List ts) { @@ -308,9 +300,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer subscriptionManagerService.onTimeSeriesUpdate(tenantId, entityId, updated, TbCallback.EMPTY); subscriptionManagerService.onTimeSeriesDelete(tenantId, entityId, deleted, TbCallback.EMPTY); - }, () -> { - return TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys); - }); + }, () -> TbSubscriptionUtils.toTimeseriesDeleteProto(tenantId, entityId, keys)); } private void addMainCallback(ListenableFuture saveFuture, final FutureCallback callback) { diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 69334b774f..151d093c4f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.msg.edge.ToEdgeSyncRequest; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -76,6 +77,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushNotificationToTransport(String targetServiceId, ToTransportMsg response, TbQueueCallback callback); + void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 7ac938f52f..5c6bf545b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -323,7 +323,8 @@ public class HashPartitionService implements PartitionService { } } - private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { + @Override + public TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b5744981bd..8e2152830e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -37,6 +37,8 @@ public interface PartitionService { TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId); + TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId); + List resolveAll(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId); boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..69b3dff1f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -160,6 +161,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } + TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 9cf18e6cb4..77f3cbcdfe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -17,6 +17,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -48,6 +50,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -66,6 +70,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); + this.toCalculatedFieldNotifications = tbQueueProvider.createToCalculatedFieldNotificationMsgProducer(); } @Override @@ -124,4 +130,14 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toEdgeEvents; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + return toCalculatedFieldNotifications; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ec31763baa..2a151d58ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -76,7 +78,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return @@ -91,4 +93,8 @@ public interface TbQueueProducerProvider { TbQueueProducer> getTbEdgeEventsMsgProducer(); + TbQueueProducer> getCalculatedFieldsMsgProducer(); + + TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e9f7773a26..6a73dce1cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -18,6 +18,8 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -47,6 +49,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toEdge; private TbQueueProducer> toEdgeNotifications; private TbQueueProducer> toEdgeEvents; + private TbQueueProducer> toCalculatedFields; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -64,6 +67,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toEdge = tbQueueProvider.createEdgeMsgProducer(); this.toEdgeNotifications = tbQueueProvider.createEdgeNotificationsMsgProducer(); this.toEdgeEvents = tbQueueProvider.createEdgeEventMsgProducer(); + this.toCalculatedFields = tbQueueProvider.createToCalculatedFieldMsgProducer(); } @Override @@ -121,4 +125,14 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + return toCalculatedFields; + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine Service!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 7960cbcf32..88b0f0b1ef 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -111,4 +112,13 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index cd4fa12df0..09c835e5fe 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import jakarta.annotation.PostConstruct; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -107,4 +108,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider return toHousekeeper; } + @Override + public TbQueueProducer> getCalculatedFieldsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getCalculatedFieldsNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + } From 2cc0d6f513bdd4f5873e62af1db51d5d6b10eab4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 23 Jan 2025 15:49:31 +0200 Subject: [PATCH 086/438] added implementations for consumer/producer methods --- ...faultTbCalculatedFieldConsumerService.java | 8 +- .../TbCalculatedFieldConsumerService.java | 15 ++++ .../src/main/resources/thingsboard.yml | 2 + .../queue/kafka/TbKafkaTopicConfigs.java | 5 ++ .../provider/KafkaMonolithQueueFactory.java | 81 +++++++++++++++++++ .../provider/KafkaTbCoreQueueFactory.java | 15 ++++ .../KafkaTbRuleEngineQueueFactory.java | 72 ++++++++++++++++- .../queue/provider/TbCoreQueueFactory.java | 8 +- .../provider/TbRuleEngineQueueFactory.java | 8 +- .../dao/sql/event/EventInsertRepository.java | 2 +- 10 files changed, 202 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 85ee01ac5f..ff4634f957 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java index 387bdd7143..bba4b1e35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCalculatedFieldConsumerService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 94c1bef71c..fdd9923292 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1619,6 +1619,8 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field topics + calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..cdd0add38b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,8 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.calculated-field:}") + private String calculatedFieldProperties; @Getter private Map coreConfigs; @@ -79,6 +81,8 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map calculatedFieldConfigs; @PostConstruct private void init() { @@ -97,6 +101,7 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..01e174023c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -54,6 +57,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -94,6 +99,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -106,6 +112,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; @@ -119,6 +126,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +141,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -490,6 +499,75 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId())); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -522,5 +600,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (edgeAdmin != null) { edgeAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..aceccf7e58 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -53,6 +54,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -79,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -107,6 +110,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -119,6 +123,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -439,6 +444,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-calculated-field-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..46b35f9acd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,6 +23,9 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; @@ -49,6 +52,7 @@ import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; import org.thingsboard.server.queue.kafka.TbKafkaSettings; import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueEdgeSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; @@ -71,6 +75,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final TbQueueCalculatedFieldSettings calculatedFieldSettings; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -81,6 +86,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -90,7 +96,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, + TbQueueEdgeSettings edgeSettings, TbQueueCalculatedFieldSettings calculatedFieldSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -101,6 +107,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.calculatedFieldSettings = calculatedFieldSettings; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -111,6 +118,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -293,6 +301,65 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.getCalculatedFieldNotificationsTopic(serviceInfoProvider.getServiceId()).getFullTopicName()); + consumerBuilder.clientId("tb-calculated-field-notifications-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId(topicService.buildTopicName("tb-calculated-field-notifications-node-") + serviceInfoProvider.getServiceId()); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(notificationAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(cfAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createCalculatedFieldStateProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -313,5 +380,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 0b3df5bccf..673ac3a434 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 76dad05393..d3c1d09399 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 9307b9e9be..bdb675cbb7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -83,7 +83,7 @@ public class EventInsertRepository { " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); insertStmtMap.put(EventType.DEBUG_CALCULATED_FIELD, "INSERT INTO " + EventType.DEBUG_CALCULATED_FIELD.getTable() + - " (id, tenant_id, tsб entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + + " (id, tenant_id, ts, entity_id, service_id, cf_id, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_args, e_result, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); } From 564162644360fddc6e37c7e29d485d72b88dc1e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 10:53:13 +0200 Subject: [PATCH 087/438] added processNotification impl --- .../cf/CalculatedFieldExecutionService.java | 14 +- ...efaultCalculatedFieldExecutionService.java | 309 ++++++++---------- .../cf/ctx/state/CalculatedFieldCtx.java | 29 +- ...faultTbCalculatedFieldConsumerService.java | 71 ++-- .../queue/DefaultTbClusterService.java | 36 +- .../queue/DefaultTbCoreConsumerService.java | 37 --- .../DefaultTelemetrySubscriptionService.java | 4 +- .../server/cluster/TbClusterService.java | 2 + .../server/common/util/ProtoUtils.java | 94 ------ .../provider/KafkaTbCoreQueueFactory.java | 16 + .../queue/provider/TbCoreQueueFactory.java | 4 +- 11 files changed, 245 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 2507546013..1ad0377e27 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; public interface CalculatedFieldExecutionService { @@ -30,18 +32,18 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(AttributesSaveRequest request); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ - void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); + void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); - void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); +// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); + void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0f8cdd4fd4..cbd5445cdf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -33,17 +33,15 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; -import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -74,7 +72,14 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -87,15 +92,12 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -112,7 +114,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -195,7 +196,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + @Override + public void pushRequestToQueue(AttributesSaveRequest request) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -223,31 +232,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ////TODO: IM to push to CF queue - return null; - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -318,19 +302,20 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback) { + public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - if (proto.getDeleted()) { + ComponentLifecycleEvent event = proto.getEvent(); + if (ComponentLifecycleEvent.DELETED.equals(event)) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (proto.getUpdated()) { + if (ComponentLifecycleEvent.UPDATED.equals(event)) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { @@ -340,14 +325,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (cf != null) { calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -492,30 +477,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(request); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } +// @Override +// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { +// try { +// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); +// +// if (proto.getLinksList().isEmpty()) { +// onTelemetryUpdate(request); +// return; +// } +// +// proto.getLinksList().forEach(ctxIdProto -> { +// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); +// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); +// +// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); +// if (!updatedTelemetry.isEmpty()) { +// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); +// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); +// } +// }); +// } catch (Exception e) { +// log.trace("Failed to process telemetry update msg: [{}]", proto, e); +// } +// } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -526,50 +511,41 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { + public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { try { TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity type update msg: [{}]", proto, e); - } - } - - @Override - public void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); - } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(entityId, profileId, callback); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + } + if (proto.getAdded()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } + if (proto.getUpdated()) { + log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); + EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); + + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); } } else { - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); + clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); } } catch (Exception e) { - log.trace("Failed to process profile entity msg: [{}]", proto, e); + log.trace("Failed to process entity update msg: [{}]", proto, e); } } @@ -582,7 +558,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() - .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId())) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -775,88 +751,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { - return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); - } - - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( - CalculatedFieldTelemetryUpdateRequest request, List links - ) { - TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); - - builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) - .setEntityType(request.getEntityId().getEntityType().name()) - .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) - .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); - - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - - for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { - builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); - } - - if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { - builder.setScope(attributeUpdateRequest.getScope().name()); - } - - for (KvEntry entry : request.getKvEntries()) { - TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); - if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { - telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); - } - if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { - telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); - } - builder.addUpdatedTelemetry(telemetryBuilder.build()); - } - - return builder.build(); - } - - private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - return attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build(); - } +// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { +// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() +// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) +// .setEntityType(ctxId.entityId().getEntityType().name()) +// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) +// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) +// .build(); +// } +// +// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { +// return TransportProtos.CalculatedFieldIdProto.newBuilder() +// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) +// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) +// .build(); +// } private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); @@ -905,6 +815,51 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + for (TsKvEntry entry : request.getEntries()) { + telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + for (AttributeKvEntry entry : request.getEntries()) { + telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + for (CalculatedFieldId cfId : calculatedFieldIds) { + CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); + calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); + calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); + telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + } + + return telemetryMsg; + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a9f999497..e4abbee4cd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; @@ -99,20 +101,35 @@ public class CalculatedFieldCtx { ); } + public boolean matches(List values, AttributeScope scope) { + return matchesAttributes(mainEntityArguments, values, scope); + } + + public boolean linkMatches(EntityId entityId, List values, AttributeScope scope) { + var map = linkedEntityArguments.get(entityId); + return map != null && matchesAttributes(map, values, scope); + } + public boolean matches(List values) { - return matches(mainEntityArguments, values); + return matchesTimeSeries(mainEntityArguments, values); } public boolean linkMatches(EntityId entityId, List values) { var map = linkedEntityArguments.get(entityId); - if (map == null) { - return false; - } else { - return matches(map, values); + return map != null && matchesTimeSeries(map, values); + } + + private static boolean matchesAttributes(Map argMap, List values, AttributeScope scope) { + for (AttributeKvEntry attrKv : values) { + ReferencedEntityKey attrKey = new ReferencedEntityKey(attrKv.getKey(), ArgumentType.ATTRIBUTE, scope); + if (argMap.containsKey(attrKey)) { + return true; + } } + return false; } - private static boolean matches(Map argMap, List values) { + private boolean matchesTimeSeries(Map argMap, List values) { for (TsKvEntry tsKv : values) { ReferencedEntityKey latestKey = new ReferencedEntityKey(tsKv.getKey(), ArgumentType.TS_LATEST, null); if (argMap.containsKey(latestKey)) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ff4634f957..4eb3f0cd89 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -24,13 +25,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -157,8 +162,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { - ToCalculatedFieldNotificationMsg notification = msg.getValue(); - + ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); + if (toCfNotification.hasComponentLifecycle()) { + forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + } else if (toCfNotification.hasEntityUpdateMsg()) { + forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + } callback.onSuccess(); } @@ -184,41 +193,29 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // -// private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldMsgProto calculatedFieldMsg, TbCallback callback) { -// var tenantId = toTenantId(calculatedFieldMsg.getTenantIdMSB(), calculatedFieldMsg.getTenantIdLSB()); -// var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldMsg.getCalculatedFieldIdMSB(), calculatedFieldMsg.getCalculatedFieldIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } -// -// private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { -// var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); -// ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); -// DonAsynchron.withCallback(future, -// __ -> callback.onSuccess(), -// t -> { -// log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); -// callback.onFailure(t); -// }); -// } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + + private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); + callback.onFailure(t); + }); + } private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 725e0fdd79..b510b813d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -69,8 +69,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -80,6 +79,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.QueueDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceDeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.ResourceUpdateMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -109,7 +110,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; import static org.thingsboard.server.common.util.ProtoUtils.toProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @@ -346,6 +346,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state) { log.trace("[{}] Processing {} state change event: {}", tenantId, entityId.getEntityType(), state); @@ -809,16 +816,23 @@ public class DefaultTbClusterService implements TbClusterService { } private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldMsgProto.Builder builder = TransportProtos.CalculatedFieldMsgProto.newBuilder(); + ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - TransportProtos.CalculatedFieldMsgProto msg = builder.build(); - pushMsgToCore(tenantId, calculatedFieldId, ToCoreMsg.newBuilder().setCalculatedFieldMsg(msg).build(), null); + builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); + builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); + builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); + TransportProtos.ComponentLifecycleEvent event; + if (added) { + event = TransportProtos.ComponentLifecycleEvent.CREATED; + } else if (updated) { + event = TransportProtos.ComponentLifecycleEvent.UPDATED; + } else { + event = TransportProtos.ComponentLifecycleEvent.DELETED; + } + builder.setEvent(event); + + pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index c598540ff2..e3a1ca22d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -702,42 +701,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldMsg(calculatedFieldMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { - var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity profile updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(TransportProtos.ProfileEntityMsgProto profileEntityMsgProto, TbCallback callback) { - var tenantId = toTenantId(profileEntityMsgProto.getTenantIdMSB(), profileEntityMsgProto.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsgProto.getEntityType(), new UUID(profileEntityMsgProto.getEntityIdMSB(), profileEntityMsgProto.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onProfileEntityMsg(profileEntityMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process profile entity message for entityId [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 71716a16e9..0f13dbb2f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -168,7 +168,9 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); - //TODO: IM to push to CF queue + DonAsynchron.withCallback(saveFuture, result -> { + calculatedFieldExecutionService.pushRequestToQueue(request); + }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 151d093c4f..b28bf73c89 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); + void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); void onDeviceProfileChange(DeviceProfile deviceProfile, DeviceProfile oldDeviceProfile, TbQueueCallback callback); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 1b743316bd..3bace4d91f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -59,7 +58,6 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -630,98 +628,6 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } - public static KvEntry fromProto(TransportProtos.TsKvProto proto) { - TransportProtos.KeyValueProto kvProto = proto.getKv(); - String key = kvProto.getKey(); - KvEntry entry = switch (kvProto.getType()) { - case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); - case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); - case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); - case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); - case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); - default -> null; - }; - return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); - } - -// public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { -// if (telemetryProto.hasAttrKv()) { -// return fromProto(telemetryProto.getAttrKv().getValue()); -// } else if (telemetryProto.hasTsKv()) { -// return fromProto(telemetryProto.getTsKv()); -// } else { -// throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); -// } -// } - - public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { - TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); - builder.setAttributeKey(key); - switch (scope) { - case CLIENT_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); - break; - case SERVER_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); - break; - case SHARED_SCOPE: - builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); - break; - default: - throw new IllegalArgumentException("Unsupported attribute scope: " + scope); - } - return builder.build(); - } - -// public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { -// return TransportProtos.AttributeKvProto.newBuilder() -// .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) -// .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) -// .build(); -// } - - public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { - TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); - builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); - switch (attributeKvEntry.getDataType()) { - case BOOLEAN: - builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) - .setHasV(true) - .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); - break; - case LONG: - builder.setType(TransportProtos.KeyValueType.LONG_V) - .setHasV(true) - .setLongV(attributeKvEntry.getLongValue().orElse(0L)); - break; - case DOUBLE: - builder.setType(TransportProtos.KeyValueType.DOUBLE_V) - .setHasV(true) - .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); - break; - case STRING: - builder.setType(TransportProtos.KeyValueType.STRING_V) - .setHasV(true) - .setStringV(attributeKvEntry.getStrValue().orElse("")); - break; - case JSON: - builder.setType(TransportProtos.KeyValueType.JSON_V) - .setHasV(true) - .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); - break; - default: - builder.setHasV(false); - throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); - } - if (attributeKvEntry.getKey() != null) { - builder.setKey(attributeKvEntry.getKey()); - } - if (attributeKvEntry.getVersion() != null) { - builder.setVersion(attributeKvEntry.getVersion()); - } - return builder.build(); - } - public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index aceccf7e58..329f1d7544 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -96,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin cfAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -138,6 +140,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); } @Override @@ -444,6 +447,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createToCalculatedFieldMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-to-calculated-field-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); + requestBuilder.admin(cfAdmin); + return requestBuilder.build(); + } + @Override public TbQueueProducer> createToCalculatedFieldNotificationMsgProducer() { TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); @@ -483,5 +496,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (vcAdmin != null) { vcAdmin.destroy(); } + if (cfAdmin != null) { + cfAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 69b3dff1f0..1ece8249f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,7 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; @@ -161,7 +161,7 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, Hous return null; } - TbQueueProducer> createToCalculatedFieldMsgProducer(); + TbQueueProducer> createToCalculatedFieldMsgProducer(); TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); From 85119d02475a5cbfb6688bc609414f2d3d5bf5e0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 11:37:32 +0200 Subject: [PATCH 088/438] WIP: cluster mode implementation --- .../cf/CalculatedFieldExecutionService.java | 4 +- ...efaultCalculatedFieldExecutionService.java | 11 +++-- ...faultTbCalculatedFieldConsumerService.java | 6 +-- .../queue/DefaultTbClusterService.java | 42 +++++++------------ .../DefaultTelemetrySubscriptionService.java | 3 +- .../src/main/resources/thingsboard.yml | 2 +- 6 files changed, 29 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 1ad0377e27..93a0ecc75f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -23,6 +23,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import java.util.List; + public interface CalculatedFieldExecutionService { /** @@ -32,7 +34,7 @@ public interface CalculatedFieldExecutionService { */ void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); - void pushRequestToQueue(AttributesSaveRequest request); + void pushRequestToQueue(AttributesSaveRequest request, List result); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index cbd5445cdf..d46a7515ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -197,14 +197,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(AttributesSaveRequest request) { + public void pushRequestToQueue(AttributesSaveRequest request, List result) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); } - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, Predicate mainEntityFilter, Predicate linkedEntityFilter, Supplier msg, FutureCallback callback) { + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); if (send) { clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); @@ -827,7 +829,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request) { + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { + //TODO: IM Use result in both methods to update the versions of telemetry/attributes. ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 4eb3f0cd89..19d6e295b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -62,9 +62,9 @@ import java.util.UUID; @Slf4j public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerService implements TbCalculatedFieldConsumerService { - @Value("${queue.calculated_fields.poll_interval}") + @Value("${queue.calculated_fields.poll_interval:25}") private long pollInterval; - @Value("${queue.calculated_fields.pack_processing_timeout}") + @Value("${queue.calculated_fields.pack_processing_timeout:60000}") private long packProcessingTimeout; @Value("${queue.calculated_fields.consumer_per_partition:true}") private boolean consumerPerPartition; @@ -102,7 +102,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_CORE)) + .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b510b813d6..3064277bf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -102,6 +102,7 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -570,7 +571,9 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -581,7 +584,6 @@ public class DefaultTbClusterService implements TbClusterService { || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.NOTIFICATION_RULE) - || entityType.equals(EntityType.CALCULATED_FIELD) ) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); @@ -594,6 +596,14 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } + if (entityType.equals(EntityType.CALCULATED_FIELD)) { + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -641,11 +651,11 @@ public class DefaultTbClusterService implements TbClusterService { if (deviceNameChanged) { gatewayNotificationsService.onDeviceUpdated(device, old); } - boolean deviceTypeChanged = !device.getType().equals(old.getType()); - if (deviceTypeChanged) { + boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + if (deviceProfileChanged) { handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } - if (deviceNameChanged || deviceTypeChanged) { + if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { @@ -802,37 +812,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { var created = oldCalculatedField == null; - broadcastEntityChangeToTransport(calculatedField.getTenantId(), calculatedField.getId(), calculatedField, null); broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendCalculatedFieldEvent(calculatedField.getTenantId(), calculatedField.getId(), created, !created, false); } @Override public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityDeleteToTransport(tenantId, calculatedFieldId, calculatedField.getName(), callback); broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); - sendCalculatedFieldEvent(tenantId, calculatedFieldId, false, false, true); - } - - private void sendCalculatedFieldEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, boolean added, boolean updated, boolean deleted) { - ComponentLifecycleMsgProto.Builder builder = ComponentLifecycleMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(TransportProtos.EntityTypeProto.CALCULATED_FIELD); - builder.setEntityIdMSB(calculatedFieldId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(calculatedFieldId.getId().getLeastSignificantBits()); - TransportProtos.ComponentLifecycleEvent event; - if (added) { - event = TransportProtos.ComponentLifecycleEvent.CREATED; - } else if (updated) { - event = TransportProtos.ComponentLifecycleEvent.UPDATED; - } else { - event = TransportProtos.ComponentLifecycleEvent.DELETED; - } - builder.setEvent(event); - - pushNotificationToCalculatedFields(tenantId, calculatedFieldId, ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(builder).build(), null); } private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 0f13dbb2f9..7f889d431f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -167,9 +167,8 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public void saveAttributesInternal(AttributesSaveRequest request) { log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); - addMainCallback(saveFuture, request.getCallback()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request); + calculatedFieldExecutionService.pushRequestToQueue(request, result); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index fdd9923292..d62aac5e6c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1750,7 +1750,7 @@ queue: # Amount of partitions used by CF microservices partitions: "${TB_QUEUE_CF_PARTITIONS:10}" # Timeout for processing a message pack by CF microservices - pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack_processing_timeout: "${TB_QUEUE_CF_PACK_PROCESSING_TIMEOUT_MS:60000}" # Enable/disable a separate consumer per partition for CF queue consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages From bd34ed5011ac6d854bd06dcd92095662bc1a16b0 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:34:18 +0200 Subject: [PATCH 089/438] Implemented calculated fields table --- .../core/http/calculated-fields.service.ts | 78 +++++++++ .../calculated-fields-table-config.ts | 157 ++++++++++++++++++ .../calculated-fields-table.component.html | 1 + .../calculated-fields-table.component.ts | 104 ++++++++++++ .../entity/entities-table.component.ts | 8 +- .../home/components/home-components.module.ts | 9 +- .../entity/entity-table-component.models.ts | 4 +- .../pages/device/device-tabs.component.html | 4 + .../app/shared/models/entity-type.models.ts | 15 +- .../assets/locale/locale.constant-en_US.json | 9 + 10 files changed, 383 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/core/http/calculated-fields.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts new file mode 100644 index 0000000000..92eae2c25a --- /dev/null +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; +import { HttpClient } from '@angular/common/http'; +import { PageData } from '@shared/models/page/page-data'; + +@Injectable({ + providedIn: 'root' +}) +// [TODO]: [Calculated fields] - implement when BE ready +export class CalculatedFieldsService { + + fieldsMock = [ + { + name: 'Calculated Field 1', + type: 'Simple', + expression: '1 + 2', + id: { + id: '1', + } + }, + { + name: 'Calculated Field 2', + type: 'Script', + expression: '${power}', + id: { + id: '2', + } + } + ]; + + constructor( + private http: HttpClient + ) { } + + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(this.fieldsMock[0]); + // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + return of(this.fieldsMock[1]); + // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + } + + public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + return of(true); + // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + } + + public getCalculatedFields(query: any, + config?: RequestConfig): Observable> { + return of({ + data: this.fieldsMock, + totalPages: 1, + totalElements: 2, + hasNext: false, + }); + // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts new file mode 100644 index 0000000000..f2e2a64b49 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -0,0 +1,157 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { TranslateService } from '@ngx-translate/core'; +import { Direction } from '@shared/models/page/sort-order'; +import { MatDialog } from '@angular/material/dialog'; +import { TimePageLink } from '@shared/models/page/page-link'; +import { Observable, of } from 'rxjs'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { DialogService } from '@core/services/dialog.service'; +import { MINUTE } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthState } from '@core/auth/auth.selectors'; +import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { EntityService } from '@core/http/entity.service'; +import { EntityDebugSettings } from '@shared/models/entity.models'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { catchError, switchMap } from 'rxjs/operators'; + +export class CalculatedFieldsTableConfig extends EntityTableConfig { + + readonly calculatedFieldsDebugPerTenantLimitsConfiguration = + getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; + readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + public entityId: EntityId = null, + private store: Store, + private viewContainerRef: ViewContainerRef, + private overlay: Overlay, + private cd: ChangeDetectorRef, + private utilsService: UtilsService, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + ) { + super(); + this.tableTitle = this.translate.instant('calculated-fields.label'); + this.detailsPanelEnabled = false; + this.selectionEnabled = true; + this.searchEnabled = true; + this.addEnabled = true; + this.entitiesDeleteEnabled = true; + this.actionsColumnTitle = ''; + this.entityType = EntityType.CALCULATED_FIELDS; + this.entityTranslations = entityTypeTranslations.get(EntityType.CALCULATED_FIELDS); + + this.entitiesFetchFunction = pageLink => this.fetchCalculatedFields(pageLink); + + this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; + + this.columns.push( + new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push( + new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push( + new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + + this.cellActionDescriptors.push( + { + name: '', + nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + icon: 'mdi:bug', + isEnabled: () => true, + iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', + onAction: ($event, entity) => this.onOpenDebugConfig($event, entity), + }, + { + name: this.translate.instant('action.edit'), + icon: 'edit', + isEnabled: () => true, + // // [TODO]: [Calculated fields] - implement edit + onAction: (_, entity) => {} + } + ); + } + + fetchCalculatedFields(pageLink: TimePageLink): Observable> { + return this.calculatedFieldsService.getCalculatedFields(pageLink); + } + + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + const { renderer, viewContainerRef } = this.getTable(); + if ($event) { + $event.stopPropagation(); + } + const trigger = $event.target as Element; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const debugStrategyPopover = this.popoverService.displayPopover(trigger, renderer, + viewContainerRef, EntityDebugSettingsPanelComponent, 'bottom', true, null, + { + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + maxDebugModeDuration: this.maxDebugModeDuration, + entityLabel: this.translate.instant('debug-settings.integration'), + ...debugSettings + }, + {}, + {}, {}, true); + debugStrategyPopover.tbComponentRef.instance.popover = debugStrategyPopover; + debugStrategyPopover.tbComponentRef.instance.onSettingsApplied.pipe(takeUntilDestroyed(this.destroyRef)).subscribe((settings: EntityDebugSettings) => { + this.onDebugConfigChanged(id.id, settings); + debugStrategyPopover.hide(); + }); + } + } + + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { + const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); + + if (!isDebugActive) { + return debugSettings?.failuresEnabled ? this.translate.instant('debug-settings.failures') : this.translate.instant('common.disabled'); + } else { + return this.durationLeft.transform(debugSettings?.allEnabledUntil) + } + } + + private isDebugActive(allEnabledUntil: number): boolean { + return allEnabledUntil > new Date().getTime(); + } + + private onDebugConfigChanged(id: string, debugSettings: EntityDebugSettings): void { + this.calculatedFieldsService.getCalculatedField(id).pipe( + switchMap(field => this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), + catchError(() => of(null)), + takeUntilDestroyed(this.destroyRef), + ).subscribe(() => this.updateData()); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html new file mode 100644 index 0000000000..38aa1487af --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts new file mode 100644 index 0000000000..a5b1ab34c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DestroyRef, + Input, + OnInit, + ViewChild, + ViewContainerRef, +} from '@angular/core'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { EntityService } from '@core/http/entity.service'; +import { DialogService } from '@core/services/dialog.service'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Overlay } from '@angular/cdk/overlay'; +import { UtilsService } from '@core/services/utils.service'; +import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; + +@Component({ + selector: 'tb-calculated-fields-table', + templateUrl: './calculated-fields-table.component.html', + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class CalculatedFieldsTableComponent implements OnInit { + + @Input() entityId: EntityId; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + this.entitiesTable.updateData(); + } + } + } + + @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; + + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + + private activeValue = false; + private dirtyValue = false; + + constructor(private calculatedFieldsService: CalculatedFieldsService, + private entityService: EntityService, + private dialogService: DialogService, + private translate: TranslateService, + private dialog: MatDialog, + private store: Store, + private overlay: Overlay, + private viewContainerRef: ViewContainerRef, + private cd: ChangeDetectorRef, + private durationLeft: DurationLeftPipe, + private popoverService: TbPopoverService, + private destroyRef: DestroyRef, + private utilsService: UtilsService) { + } + + ngOnInit() { + this.dirtyValue = !this.activeValue; + + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.entityService, + this.dialogService, + this.translate, + this.dialog, + this.entityId, + this.store, + this.viewContainerRef, + this.overlay, + this.cd, + this.utilsService, + this.durationLeft, + this.popoverService, + this.destroyRef + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 9bb5fe4d8a..3c723f8080 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -25,8 +25,10 @@ import { OnChanges, OnDestroy, OnInit, + Renderer2, SimpleChanges, - ViewChild + ViewChild, + ViewContainerRef, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -141,7 +143,9 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa private router: Router, private elementRef: ElementRef, private fb: FormBuilder, - private zone: NgZone) { + private zone: NgZone, + public viewContainerRef: ViewContainerRef, + public renderer: Renderer2) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 32e509e842..1a6b9c08e0 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -183,6 +183,8 @@ import { } from '@home/components/dashboard-page/layout/select-dashboard-breakpoint.component'; import { EntityChipsComponent } from '@home/components/entity/entity-chips.component'; import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; +import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; +import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; @NgModule({ declarations: @@ -326,7 +328,8 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], imports: [ CommonModule, @@ -463,11 +466,13 @@ import { DashboardViewComponent } from '@home/components/dashboard-view/dashboar RateLimitsDetailsDialogComponent, SendNotificationButtonComponent, EntityChipsComponent, - DashboardViewComponent + DashboardViewComponent, + CalculatedFieldsTableComponent, ], providers: [ WidgetComponentService, CustomDialogService, + DurationLeftPipe, {provide: EMBED_DASHBOARD_DIALOG_TOKEN, useValue: EmbedDashboardDialogComponent}, {provide: COMPLEX_FILTER_PREDICATE_DIALOG_COMPONENT_TOKEN, useValue: ComplexFilterPredicateDialogComponent}, {provide: DASHBOARD_PAGE_COMPONENT_TOKEN, useValue: DashboardPageComponent}, diff --git a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts index b6f634195e..8c4f856609 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entity-table-component.models.ts @@ -20,7 +20,7 @@ import { SafeHtml } from '@angular/platform-browser'; import { PageLink } from '@shared/models/page/page-link'; import { Timewindow } from '@shared/models/time/time.models'; import { EntitiesDataSource } from '@home/models/datasource/entity-datasource'; -import { ElementRef, EventEmitter } from '@angular/core'; +import { ElementRef, EventEmitter, Renderer2, ViewContainerRef } from '@angular/core'; import { TbAnchorComponent } from '@shared/components/tb-anchor.component'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -64,6 +64,8 @@ export interface IEntitiesTableComponent { paginator: MatPaginator; sort: MatSort; route: ActivatedRoute; + viewContainerRef: ViewContainerRef; + renderer: Renderer2; addEnabled(): boolean; clearSelection(): void; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 5e30694719..357bb587cc 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index e1b59a9243..5b540a6c54 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -49,7 +49,8 @@ export enum EntityType { OAUTH2_CLIENT = 'OAUTH2_CLIENT', DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', - MOBILE_APP = 'MOBILE_APP' + MOBILE_APP = 'MOBILE_APP', + CALCULATED_FIELDS = 'CALCULATED_FIELDS', } export enum AliasEntityType { @@ -478,6 +479,18 @@ export const entityTypeTranslations = new MapAre you sure you want to leave this page?", @@ -1027,6 +1034,8 @@ "city-max-length": "Specified city should be less than 256" }, "common": { + "name": "Name", + "type": "Type", "username": "Username", "password": "Password", "enter-username": "Enter username", From a652b31d7f880b1c13734c26c5f331e971a524e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 24 Jan 2025 15:41:00 +0200 Subject: [PATCH 090/438] implementation processing notification --- .../service/cf/CalculatedFieldCache.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 7 +- .../cf/DefaultCalculatedFieldCache.java | 32 ++++ ...efaultCalculatedFieldExecutionService.java | 164 +++++++++++------- .../entitiy/EntityStateSourcingListener.java | 21 +-- .../DefaultSystemDataLoaderService.java | 7 +- ...faultTbCalculatedFieldConsumerService.java | 2 +- .../queue/DefaultTbClusterService.java | 74 ++++---- .../queue/DefaultTbCoreConsumerService.java | 41 ----- .../processing/AbstractConsumerService.java | 4 + .../TbRuleEngineQueueConsumerManager.java | 14 +- .../DefaultTelemetrySubscriptionService.java | 2 - .../server/cluster/TbClusterService.java | 8 +- .../server/common/util/ProtoUtils.java | 15 ++ common/proto/src/main/proto/queue.proto | 4 - .../server/dao/service/EntityServiceTest.java | 5 +- 16 files changed, 211 insertions(+), 194 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ea55894432..1ee1d4d562 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -42,6 +41,10 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void evictProfile(TenantId tenantId, EntityId entityId); + + void evictEntity(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..836af03e5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,8 +20,9 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; import java.util.List; @@ -42,9 +43,9 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); -// void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..4278e74845 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,11 +22,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -172,6 +177,33 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void evictProfile(TenantId tenantId, EntityId entityId) { + log.debug("[{}] evict entity profile from cache.", entityId); + profileEntities.remove(entityId); + } + + @Override + public void evictEntity(TenantId tenantId, EntityId entityId) { + calculatedFieldFetchLock.lock(); + try { + profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); + if (EntityType.ASSET.equals(entityId.getEntityType())) { + Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); + if (asset != null) { + profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } else { + Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); + if (device != null) { + profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); + } + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..beb347d29f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -36,6 +36,7 @@ import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; @@ -72,14 +73,17 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; @@ -92,7 +96,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -334,7 +340,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas initializeStateForEntity(calculatedFieldCtx, entityId, callback); } case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantICalculatedFieldMsgProtod=[{}], profileId=[{}]", tenantId, entityId); + log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -401,8 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { + public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { @@ -418,15 +425,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); - clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() - .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); } } } catch (Exception e) { @@ -479,30 +483,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } -// @Override -// public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { -// try { -// CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); -// -// if (proto.getLinksList().isEmpty()) { -// onTelemetryUpdate(request); -// return; -// } -// -// proto.getLinksList().forEach(ctxIdProto -> { -// CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); -// CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); -// -// Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); -// if (!updatedTelemetry.isEmpty()) { -// EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); -// executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); -// } -// }); -// } catch (Exception e) { -// log.trace("Failed to process telemetry update msg: [{}]", proto, e); -// } -// } + @Override + public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(proto, callback); + return; + } + + proto.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); + + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } + } private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); @@ -753,23 +757,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } -// private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { -// return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() -// .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) -// .setEntityType(ctxId.entityId().getEntityType().name()) -// .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) -// .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) -// .build(); -// } -// -// private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { -// return TransportProtos.CalculatedFieldIdProto.newBuilder() -// .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) -// .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) -// .build(); -// } - private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); @@ -821,22 +808,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); - for (TsKvEntry entry : request.getEntries()) { - telemetryMsg.addTsData(ProtoUtils.toTsKvProto(entry)); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); return msg.build(); } - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List result) { - //TODO: IM Use result in both methods to update the versions of telemetry/attributes. + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - for (AttributeKvEntry entry : request.getEntries()) { - telemetryMsg.addAttrData(ProtoUtils.toProto(entry)); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -854,15 +847,66 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); for (CalculatedFieldId cfId : calculatedFieldIds) { - CalculatedFieldIdProto.Builder calculatedFieldIdProto = CalculatedFieldIdProto.newBuilder(); - calculatedFieldIdProto.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()); - calculatedFieldIdProto.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()); - telemetryMsg.addPreviousCalculatedFields(calculatedFieldIdProto.build()); + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); } return telemetryMsg; } + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, scope, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + } + private static TbQueueCallback wrap(FutureCallback callback) { if (callback != null) { return new FutureCallbackWrapper(callback); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index ab8246ccf5..4703ed1606 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,7 +33,6 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -88,7 +87,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -123,9 +122,6 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } - case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); - } default -> { } } @@ -150,7 +146,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -191,10 +187,6 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } - case CALCULATED_FIELD -> { - CalculatedField calculatedField = (CalculatedField) event.getEntity(); - tbClusterService.onCalculatedFieldDeleted(tenantId, calculatedField, null); - } default -> { } } @@ -275,15 +267,6 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { - CalculatedField calculatedField = (CalculatedField) entity; - CalculatedField oldCalculatedField = null; - if (oldEntity instanceof CalculatedField) { - oldCalculatedField = (CalculatedField) oldEntity; - } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField); - } - private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index fd8a59c392..5b0c790215 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.mobile.app.MobileApp; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; @@ -98,9 +99,9 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.notification.NotificationSettingsService; import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.mobile.MobileAppDao; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -308,7 +309,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { jwtSettingsService.saveJwtSettings(jwtSettings); } - List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE,0)).getData(); + List mobiles = mobileAppDao.findByTenantId(TenantId.SYS_TENANT_ID, null, new PageLink(Integer.MAX_VALUE, 0)).getData(); if (CollectionUtils.isNotEmpty(mobiles)) { mobiles.stream() .filter(mobileApp -> !validateKeyLength(mobileApp.getAppSecret())) @@ -571,7 +572,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private void save(DeviceId deviceId, String key, boolean value) { if (persistActivityToTelemetry) { - ListenableFuture saveFuture = tsService.save( + ListenableFuture saveFuture = tsService.save( TenantId.SYS_TENANT_ID, deviceId, Collections.singletonList(new BasicTsKvEntry(System.currentTimeMillis(), new BooleanDataEntry(key, value))), 0L); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..3df2653df6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -171,7 +171,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer callback.onSuccess(); } -// private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { + // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { // var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); // var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); // var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 3064277bf4..04d6a53401 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,12 +38,10 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -347,6 +345,13 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } + @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); @@ -409,7 +414,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true); + handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -418,7 +423,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), true, true); + handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -653,13 +658,13 @@ public class DefaultTbClusterService implements TbClusterService { } boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); } } else { - handleProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false); + handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); } broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); @@ -673,10 +678,10 @@ public class DefaultTbClusterService implements TbClusterService { if (old != null) { boolean assetTypeChanged = !asset.getType().equals(old.getType()); if (assetTypeChanged) { - handleEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); } } else { - handleProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false); + handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); } broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -809,52 +814,41 @@ public class DefaultTbClusterService implements TbClusterService { } } - @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField) { - var created = oldCalculatedField == null; - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); } - @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); } - private void handleEntityProfileUpdatedEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - TransportProtos.EntityProfileUpdateMsgProto.Builder builder = TransportProtos.EntityProfileUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - TransportProtos.EntityProfileUpdateMsgProto msg = builder.build(); - - broadcastToCore(ToCoreNotificationMsg.newBuilder().setEntityProfileUpdateMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setEntityProfileUpdateMsg(msg).build(), null); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { + handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); } - private void handleProfileEntityEvent(TenantId tenantId, EntityId entityId, EntityId profileId, boolean added, boolean deleted) { - TransportProtos.ProfileEntityMsgProto.Builder builder = TransportProtos.ProfileEntityMsgProto.newBuilder(); + private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { + TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - builder.setEntityProfileType(profileId.getEntityType().name()); - builder.setProfileIdMSB(profileId.getId().getMostSignificantBits()); - builder.setProfileIdLSB(profileId.getId().getLeastSignificantBits()); + if (oldProfileId != null) { + builder.setEntityProfileType(oldProfileId.getEntityType().name()); + builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); + } + if (newProfileId != null) { + builder.setEntityProfileType(newProfileId.getEntityType().name()); + builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); + builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); + } builder.setAdded(added); + builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.ProfileEntityMsgProto msg = builder.build(); + TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - broadcastToCore(ToCoreNotificationMsg.newBuilder().setProfileEntityMsg(msg).build()); - pushMsgToCore(tenantId, entityId, ToCoreMsg.newBuilder().setProfileEntityMsg(msg).build(), null); + pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..e7bde845f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -88,7 +86,6 @@ import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -110,7 +107,6 @@ import org.thingsboard.server.service.ws.notification.sub.NotificationRequestUpd import org.thingsboard.server.service.ws.notification.sub.NotificationUpdate; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -153,14 +149,12 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; private volatile ListeningExecutorService deviceActivityEventsExecutor; - private volatile ListeningExecutorService calculatedFieldsExecutor; public DefaultTbCoreConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, @@ -183,7 +177,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) @@ -319,12 +310,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); - if (added) { - entitiesByProfile.add(entityId); - } else { - entitiesByProfile.remove(entityId); - } - } - private void forwardToSubMgrService(SubscriptionMgrMsgProto msg, TbCallback callback) { if (msg.hasSubEvent()) { TbEntitySubEventProto subEvent = msg.getSubEvent(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index dac35bfc5c..5b1e5d7d79 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,12 +178,16 @@ public abstract class AbstractConsumerService onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index b28bf73c89..d2fbcddfdf 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.EdgeId; @@ -40,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -79,6 +79,8 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback); + void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, TransportProtos.ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback); void broadcastEntityStateChangeEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent state); @@ -125,8 +127,4 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField); - - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, TbQueueCallback callback); - } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 3bace4d91f..6a40f13688 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; @@ -628,6 +629,20 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { return TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 288c923aaa..ede796b12b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1591,8 +1591,6 @@ message ToCoreMsg { DeviceConnectProto deviceConnectMsg = 50; DeviceDisconnectProto deviceDisconnectMsg = 51; DeviceInactivityProto deviceInactivityMsg = 52; -// CalculatedFieldMsgProto calculatedFieldMsg = 53; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1612,8 +1610,6 @@ message ToCoreNotificationMsg { FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; -// EntityProfileUpdateMsgProto entityProfileUpdateMsg = 51; -// ProfileEntityMsgProto profileEntityMsg = 52; } /* Messages to Edge queue that are handled by ThingsBoard Core Service */ diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index a26f345fb7..8c93245837 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -1824,7 +1825,7 @@ public class EntityServiceTest extends AbstractServiceTest { } } - List> timeseriesFutures = new ArrayList<>(); + List> timeseriesFutures = new ArrayList<>(); for (int i = 0; i < devices.size(); i++) { Device device = devices.get(i); timeseriesFutures.add(saveLongTimeseries(device.getId(), "temperature", temperatures.get(i))); @@ -2430,7 +2431,7 @@ public class EntityServiceTest extends AbstractServiceTest { return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); } - private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { + private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { TsKvEntity tsKv = new TsKvEntity(); tsKv.setStrKey(key); tsKv.setDoubleValue(value); From c332e7373f507199552a814c424dd98b07dd4233 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jan 2025 15:45:00 +0200 Subject: [PATCH 091/438] WIP: CalculatedFieldConsumer refactoring --- .../cf/CalculatedFieldExecutionService.java | 6 ++ ...efaultCalculatedFieldExecutionService.java | 13 +++ ...faultTbCalculatedFieldConsumerService.java | 85 ++++++++++++++++++- .../queue/DefaultTbCoreConsumerService.java | 13 +-- .../queue/DefaultTbEdgeConsumerService.java | 13 +-- .../service/queue/PendingMsgHolder.java | 24 ++++++ 6 files changed, 133 insertions(+), 21 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 93a0ecc75f..893dba92a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,6 +20,8 @@ import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; @@ -36,6 +38,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + + void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d46a7515ef..2638f22254 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -234,6 +235,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return send; } + @Override + public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + + callback.onSuccess(); + } + + @Override + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + + callback.onSuccess(); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 19d6e295b5..b36b34dd94 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,8 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -32,12 +34,19 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -50,12 +59,20 @@ import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import org.thingsboard.server.service.queue.DefaultTbCoreConsumerService.PendingMsgHolder; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; +import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Service @TbRuleEngineComponent @@ -132,7 +149,46 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { - + List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); + ConcurrentMap> pendingMap = orderedMsgList.stream().collect( + Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); + CountDownLatch processingTimeoutLatch = new CountDownLatch(1); + TbPackProcessingContext> ctx = new TbPackProcessingContext<>( + processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); + Future packSubmitFuture = consumersExecutor.submit(() -> { + orderedMsgList.forEach((element) -> { + UUID id = element.getUuid(); + TbProtoQueueMsg msg = element.getMsg(); + log.trace("[{}] Creating main callback for message: {}", id, msg.getValue()); + TbCallback callback = new TbPackCallback<>(id, ctx); + try { + ToCalculatedFieldMsg toCfMsg = msg.getValue(); + pendingMsgHolder.setMsg(toCfMsg); + if (toCfMsg.hasTelemetryMsg()) { + log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + } else if (toCfMsg.hasLinkedTelemetryMsg()) { + log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); + forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } + } catch (Throwable e) { + log.warn("[{}] Failed to process message: {}", id, msg, e); + callback.onFailure(e); + } + }); + }); + if (!processingTimeoutLatch.await(packProcessingTimeout, TimeUnit.MILLISECONDS)) { + if (!packSubmitFuture.isDone()) { + packSubmitFuture.cancel(true); + log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); + } + if (log.isDebugEnabled()) { + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + } + ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); + } + consumer.commit(); } @Override @@ -193,6 +249,33 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // } // + + private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + + } + + private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); + var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); + DonAsynchron.withCallback(future, + __ -> callback.onSuccess(), + t -> { + log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); + callback.onFailure(t); + }); + } + private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e3a1ca22d3..ca42298742 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -269,7 +269,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future packSubmitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -278,7 +278,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService(id, ctx); try { ToCoreMsg toCoreMsg = msg.getValue(); - pendingMsgHolder.setToCoreMsg(toCoreMsg); + pendingMsgHolder.setMsg(toCoreMsg); if (toCoreMsg.hasToSubscriptionMgrMsg()) { log.trace("[{}] Forwarding message to subscription manager service {}", id, toCoreMsg.getToSubscriptionMgrMsg()); forwardToSubMgrService(toCoreMsg.getToSubscriptionMgrMsg(), callback); @@ -335,8 +335,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); @@ -346,12 +345,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> ctx = new TbPackProcessingContext<>( processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); - PendingMsgHolder pendingMsgHolder = new PendingMsgHolder(); + PendingMsgHolder pendingMsgHolder = new PendingMsgHolder<>(); Future submitFuture = consumersExecutor.submit(() -> { orderedMsgList.forEach((element) -> { UUID id = element.getUuid(); @@ -145,7 +145,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService(id, ctx); try { ToEdgeMsg toEdgeMsg = msg.getValue(); - pendingMsgHolder.setToEdgeMsg(toEdgeMsg); + pendingMsgHolder.setMsg(toEdgeMsg); if (toEdgeMsg.hasEdgeNotificationMsg()) { pushNotificationToEdge(toEdgeMsg.getEdgeNotificationMsg(), 0, packProcessingRetries, callback); } @@ -161,20 +161,13 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); } - private static class PendingMsgHolder { - @Getter - @Setter - private volatile ToEdgeMsg toEdgeMsg; - } - @Override protected ServiceType getServiceType() { return ServiceType.TB_CORE; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java new file mode 100644 index 0000000000..8793e45da6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/PendingMsgHolder.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import lombok.Getter; +import lombok.Setter; + +public class PendingMsgHolder { + @Getter @Setter + private volatile T msg; +} From b169dfab2793ef1b0721f1b4b4ab7545823bdf81 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 15:46:48 +0200 Subject: [PATCH 092/438] added license headers --- .../calculated-fields-table.component.html | 17 ++++++++++++++ .../calculated-fields-table.component.scss | 22 +++++++++++++++++++ .../calculated-fields-table.component.ts | 1 + 3 files changed, 40 insertions(+) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html index 38aa1487af..d627e16ce9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.html @@ -1 +1,18 @@ + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss new file mode 100644 index 0000000000..ea3f7d90b7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + tb-entities-table { + .mat-drawer-container { + background-color: white; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index a5b1ab34c1..5449bce5a7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -42,6 +42,7 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; @Component({ selector: 'tb-calculated-fields-table', templateUrl: './calculated-fields-table.component.html', + styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) export class CalculatedFieldsTableComponent implements OnInit { From be4ea19b91636d2e9c1e927a4776c954972ebc8c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:20:55 +0200 Subject: [PATCH 093/438] added calculated fields typing --- .../core/http/calculated-fields.service.ts | 21 ++++++--- .../calculated-fields-table-config.ts | 21 ++++----- .../pages/device/device-tabs.component.html | 2 +- .../shared/models/calculated-field.models.ts | 44 +++++++++++++++++++ .../app/shared/models/entity-type.models.ts | 8 ++-- .../shared/models/id/calculated-field-id.ts | 26 +++++++++++ .../assets/locale/locale.constant-en_US.json | 3 +- 7 files changed, 103 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/shared/models/calculated-field.models.ts create mode 100644 ui-ngx/src/app/shared/models/id/calculated-field-id.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 92eae2c25a..66e0833128 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,6 +19,7 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; +import { CalculatedField } from '@shared/models/calculated-field.models'; @Injectable({ providedIn: 'root' @@ -30,7 +31,11 @@ export class CalculatedFieldsService { { name: 'Calculated Field 1', type: 'Simple', - expression: '1 + 2', + configuration: { + expression: '1 + 2', + type: 'SIMPLE', + }, + entityId: '1', id: { id: '1', } @@ -38,23 +43,27 @@ export class CalculatedFieldsService { { name: 'Calculated Field 2', type: 'Script', - expression: '${power}', + entityId: '2', + configuration: { + expression: '${power}', + type: 'SIMPLE', + }, id: { id: '2', } } - ]; + ] as any[]; constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { + public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { + public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); } @@ -65,7 +74,7 @@ export class CalculatedFieldsService { } public getCalculatedFields(query: any, - config?: RequestConfig): Observable> { + config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, totalPages: 1, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f2e2a64b49..22f742b9d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,8 +39,9 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, switchMap } from 'rxjs/operators'; +import { CalculatedField } from '@shared/models/calculated-field.models'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; @@ -62,31 +63,31 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; this.columns.push( - new EntityTableColumn('name', 'common.name', '33%')); + new EntityTableColumn('name', 'common.name', '33%')); this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); + new EntityTableColumn('type', 'common.type', '50px')); this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%')); + new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); this.cellActionDescriptors.push( { name: '', - nameFunction: (entity) => this.getDebugConfigLabel(entity?.debugSettings), + nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), icon: 'mdi:bug', isEnabled: () => true, iconFunction: ({ debugSettings }) => this.isDebugActive(debugSettings?.allEnabledUntil) || debugSettings?.failuresEnabled ? 'mdi:bug' : 'mdi:bug-outline', @@ -102,11 +103,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: TimePageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(pageLink); } - onOpenDebugConfig($event: Event, { debugSettings = {}, id }: any): void { + onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { const { renderer, viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 357bb587cc..ab59f412cd 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -33,7 +33,7 @@ + label="{{ 'entity.type-calculated-fields' | translate }}" #calculatedFieldsTab="matTab"> , HasVersion, HasTenantId { + entityId: string; + type: CalculatedFieldType; + name: string; + debugSettings?: EntityDebugSettings; + externalId?: string; + createdTime?: number; + configuration: CalculatedFieldConfiguration; +} + +export enum CalculatedFieldType { + SIMPLE = 'SIMPLE', + COMPLEX = 'COMPLEX', +} + +export interface CalculatedFieldConfiguration { + type: CalculatedFieldConfigType; + expression: string; + arguments: Record; +} + +export enum CalculatedFieldConfigType { + SIMPLE = 'SIMPLE', + SCRIPT = 'SCRIPT', +} diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index 5b540a6c54..ee39be0f27 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -50,7 +50,7 @@ export enum EntityType { DOMAIN = 'DOMAIN', MOBILE_APP_BUNDLE = 'MOBILE_APP_BUNDLE', MOBILE_APP = 'MOBILE_APP', - CALCULATED_FIELDS = 'CALCULATED_FIELDS', + CALCULATED_FIELD = 'CALCULATED_FIELD', } export enum AliasEntityType { @@ -481,10 +481,10 @@ export const entityTypeTranslations = new Map Date: Fri, 24 Jan 2025 17:24:56 +0200 Subject: [PATCH 094/438] adjusted typing --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 66e0833128..72c4f6c12c 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -20,6 +20,7 @@ import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' @@ -73,7 +74,7 @@ export class CalculatedFieldsService { // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(query: any, + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { return of({ data: this.fieldsMock, @@ -81,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${query.toQuery()}`, + // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From 41b0963884edc21bb28569701d66fd269e7daa9e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 17:34:25 +0200 Subject: [PATCH 095/438] updated endpoint --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 72c4f6c12c..5df4a84949 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -61,17 +61,17 @@ export class CalculatedFieldsService { public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { return of(this.fieldsMock[1]); - // return this.http.post('/api/calculated-field', calculatedField, defaultHttpOptionsFromConfig(config)); + // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { return of(true); - // return this.http.delete(`/api/calculated-field/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } public getCalculatedFields(pageLink: PageLink, @@ -82,7 +82,7 @@ export class CalculatedFieldsService { totalElements: 2, hasNext: false, }); - // return this.http.get>(`/api/calculated-field${pageLink.toQuery()}`, + // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, // defaultHttpOptionsFromConfig(config)); } } From ea7e6797edba6e113c581d5650870d7029d0af25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 24 Jan 2025 18:11:18 +0200 Subject: [PATCH 096/438] Implemented set entityId --- .../calculated-fields-table.component.ts | 22 ++++++++++++++----- .../shared/models/calculated-field.models.ts | 5 +---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 5449bce5a7..f98d24bd52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -47,14 +47,23 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; }) export class CalculatedFieldsTableComponent implements OnInit { - @Input() entityId: EntityId; + @Input() + set entityId(entityId: EntityId) { + if (this.entityIdValue !== entityId) { + this.entityIdValue = entityId; + this.entitiesTable.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.hasInitialized = true; + } + } + } @Input() set active(active: boolean) { if (this.activeValue !== active) { this.activeValue = active; - if (this.activeValue && this.dirtyValue) { - this.dirtyValue = false; + if (this.activeValue && this.hasInitialized) { + this.hasInitialized = false; this.entitiesTable.updateData(); } } @@ -65,7 +74,8 @@ export class CalculatedFieldsTableComponent implements OnInit { calculatedFieldsTableConfig: CalculatedFieldsTableConfig; private activeValue = false; - private dirtyValue = false; + private hasInitialized = false; + private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, private entityService: EntityService, @@ -83,7 +93,7 @@ export class CalculatedFieldsTableComponent implements OnInit { } ngOnInit() { - this.dirtyValue = !this.activeValue; + this.hasInitialized = !this.activeValue; this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, @@ -91,7 +101,7 @@ export class CalculatedFieldsTableComponent implements OnInit { this.dialogService, this.translate, this.dialog, - this.entityId, + this.entityIdValue, this.store, this.viewContainerRef, this.overlay, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 714303e286..253bc58f39 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -17,13 +17,10 @@ import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/ent import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; -export interface CalculatedField extends BaseData, HasVersion, HasTenantId { - entityId: string; +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { type: CalculatedFieldType; - name: string; debugSettings?: EntityDebugSettings; externalId?: string; - createdTime?: number; configuration: CalculatedFieldConfiguration; } From 0e1cd69e34645ce1acd3401d85d45ef5f6fe43a2 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 27 Jan 2025 15:55:40 +0200 Subject: [PATCH 097/438] updated cf consumer --- .../service/cf/CalculatedFieldCache.java | 4 - .../cf/CalculatedFieldExecutionService.java | 8 +- .../cf/DefaultCalculatedFieldCache.java | 32 ---- ...efaultCalculatedFieldExecutionService.java | 143 ++++++++++-------- .../entitiy/EntityStateSourcingListener.java | 22 ++- ...faultTbCalculatedFieldConsumerService.java | 63 ++++---- .../queue/DefaultTbClusterService.java | 47 +++++- .../processing/AbstractConsumerService.java | 4 - .../server/cluster/TbClusterService.java | 5 + .../server/common/util/ProtoUtils.java | 13 +- common/proto/src/main/proto/queue.proto | 2 + .../provider/TbCoreQueueProducerProvider.java | 3 +- 12 files changed, 188 insertions(+), 158 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 1ee1d4d562..ff3bda5da5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -41,10 +41,6 @@ public interface CalculatedFieldCache { Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); - void evictProfile(TenantId tenantId, EntityId entityId); - - void evictEntity(TenantId tenantId, EntityId entityId); - void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..7668acba4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; @@ -30,6 +31,7 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * * @param request - telemetry save request; * @param request - telemetry save result; */ @@ -37,6 +39,8 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); + void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); @@ -47,10 +51,6 @@ public interface CalculatedFieldExecutionService { void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..7e841a0cf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,16 +22,11 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -177,33 +172,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } - @Override - public void evictProfile(TenantId tenantId, EntityId entityId) { - log.debug("[{}] evict entity profile from cache.", entityId); - profileEntities.remove(entityId); - } - - @Override - public void evictEntity(TenantId tenantId, EntityId entityId) { - calculatedFieldFetchLock.lock(); - try { - profileEntities.forEach((profile, entityIds) -> entityIds.remove(entityId)); - if (EntityType.ASSET.equals(entityId.getEntityType())) { - Asset asset = assetService.findAssetById(tenantId, (AssetId) entityId); - if (asset != null) { - profileEntities.computeIfAbsent(asset.getAssetProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } else { - Device device = deviceService.findDeviceById(tenantId, (DeviceId) entityId); - if (device != null) { - profileEntities.computeIfAbsent(device.getDeviceProfileId(), profileId -> new HashSet<>()).add(entityId); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..2a4d42ddea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -66,7 +66,6 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -120,6 +119,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; +import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -242,14 +243,59 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(msg); + EntityId entityId = request.getEntityId(); + + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + TenantId tenantId = request.getTenantId(); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + + if (tpi.isMyPartition()) { + + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); - callback.onSuccess(); + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(request, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg, ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); + }); + } + } else { + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(msg).build(), null); + } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } } @Override public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + try { + CalculatedFieldTelemetryUpdateRequest request = fromProto(linkedMsg.getMsg()); + + if (linkedMsg.getLinksList().isEmpty()) { + onTelemetryMsg(linkedMsg.getMsg(), callback); + return; + } + + linkedMsg.getLinksList().forEach(ctxIdProto -> { + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - callback.onSuccess(); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); + if (!updatedTelemetry.isEmpty()) { + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", linkedMsg, e); + } } @Override @@ -263,7 +309,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Consumer resolvePartition = entityId -> { TopicPartitionInfo tpi; try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); + tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } @@ -378,6 +424,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + @Override + public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { + EntityId entityId = calculatedField.getEntityId(); + ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); + switch (entityId.getEntityType()) { + case ASSET, DEVICE -> { + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); + } + case ASSET_PROFILE, DEVICE_PROFILE -> { + Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() + .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) + .collect(Collectors.toSet()); + tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); + } + default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() + + "' does not support calculated fields."); + } + } + private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; @@ -418,38 +484,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return entityIdChanged || typeChanged || argumentsChanged; } - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { @@ -483,7 +517,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); + TopicPartitionInfo targetEntityTpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntity); if (targetEntityTpi.isMyPartition()) { Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { @@ -495,31 +529,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto, callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() @@ -534,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); if (tpi.isMyPartition()) { log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); if (proto.getDeleted()) { @@ -824,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { long tsVersion = versions.get(i); - TsKvProto tsProto = ProtoUtils.toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); telemetryMsg.addTsData(tsProto); } msg.setTelemetryMsg(telemetryMsg.build()); @@ -858,8 +867,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 4703ed1606..95b340c362 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceId; @@ -87,7 +88,7 @@ public class EntityStateSourcingListener { case ASSET -> { onAssetUpdate(event.getEntity(), event.getOldEntity()); } - case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, lifecycleEvent); } case RULE_CHAIN -> { @@ -122,6 +123,9 @@ public class EntityStateSourcingListener { ApiUsageState apiUsageState = (ApiUsageState) event.getEntity(); tbClusterService.onApiStateChange(apiUsageState, null); } + case CALCULATED_FIELD -> { + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + } default -> { } } @@ -146,7 +150,7 @@ public class EntityStateSourcingListener { Asset asset = (Asset) event.getEntity(); tbClusterService.onAssetDeleted(tenantId, asset, null); } - case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE, CALCULATED_FIELD -> { + case ASSET_PROFILE, ENTITY_VIEW, CUSTOMER, EDGE, NOTIFICATION_RULE -> { tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityId, ComponentLifecycleEvent.DELETED); } case NOTIFICATION_REQUEST -> { @@ -187,6 +191,11 @@ public class EntityStateSourcingListener { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); tbClusterService.onResourceDeleted(tbResource, null); } + case CALCULATED_FIELD -> { + CalculatedField calculatedField = (CalculatedField) event.getEntity(); + ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); + tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + } default -> { } } @@ -267,6 +276,15 @@ public class EntityStateSourcingListener { } } + private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + CalculatedField calculatedField = (CalculatedField) entity; + CalculatedField oldCalculatedField = null; + if (oldEntity instanceof CalculatedField) { + oldCalculatedField = (CalculatedField) oldEntity; + } + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + } + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { String data = JacksonUtil.toString(JacksonUtil.valueToTree(assignedDevice)); if (data != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..bd40784e8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -34,19 +32,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -170,6 +165,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + } else if (toCfMsg.hasComponentLifecycleMsg()) { + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); + forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); + } else if (toCfMsg.hasEntityUpdateMsg()) { + log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); + forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,36 +220,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); } callback.onSuccess(); } - // private void processEntityProfileUpdateMsg(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg) { -// var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); -// var oldProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getOldProfileIdMSB(), profileUpdateMsg.getOldProfileIdLSB())); -// var newProfile = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getNewProfileIdMSB(), profileUpdateMsg.getNewProfileIdLSB())); -// calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); -// calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); -// } -// -// private void processProfileEntityMsg(TransportProtos.ProfileEntityMsgProto profileEntityMsg) { -// var tenantId = toTenantId(profileEntityMsg.getTenantIdMSB(), profileEntityMsg.getTenantIdLSB()); -// var entityId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityType(), new UUID(profileEntityMsg.getEntityIdMSB(), profileEntityMsg.getEntityIdLSB())); -// var profileId = EntityIdFactory.getByTypeAndUuid(profileEntityMsg.getEntityProfileType(), new UUID(profileEntityMsg.getProfileIdMSB(), profileEntityMsg.getProfileIdLSB())); -// boolean added = profileEntityMsg.getAdded(); -// Set entitiesByProfile = calculatedFieldCache.getEntitiesByProfile(tenantId, profileId); -// if (added) { -// entitiesByProfile.add(entityId); -// } else { -// entitiesByProfile.remove(entityId); -// } -// } -// - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); @@ -275,7 +253,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +265,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); @@ -299,6 +277,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } + private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { + var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); + var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); + if (entityUpdateMsg.getAdded()) { + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } else if (entityUpdateMsg.getDeleted()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + } else if (entityUpdateMsg.getUpdated()) { + var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); + var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); + calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); + calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); + } + } + private void throwNotHandled(Object msg, TbCallback callback) { log.warn("Message not handled: {}", msg); callback.onFailure(new RuntimeException("Message not handled!")); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 04d6a53401..b1a4b1859e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -38,10 +38,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -68,6 +70,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -95,6 +98,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -145,6 +149,10 @@ public class DefaultTbClusterService implements TbClusterService { @Lazy private OtaPackageStateService otaPackageStateService; + @Autowired + @Lazy + private CalculatedFieldExecutionService calculatedFieldExecutionService; + private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; private final TbAssetProfileCache assetProfileCache; @@ -342,21 +350,21 @@ public class DefaultTbClusterService implements TbClusterService { public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineMsgs.incrementAndGet(); } @Override public void pushMsgToCalculatedFields(TopicPartitionInfo tpi, UUID msgId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(msgId, msg), callback); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + toRuleEngineMsgs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS } @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toCoreMsgs.incrementAndGet(); + toRuleEngineNfs.incrementAndGet(); } @Override @@ -686,6 +694,20 @@ public class DefaultTbClusterService implements TbClusterService { broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } + @Override + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { + var created = oldCalculatedField == null; + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + @Override + public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { + CalculatedFieldId calculatedFieldId = calculatedField.getId(); + calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); + broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + } + @Override public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { if (!edgesEnabled) { @@ -827,7 +849,7 @@ public class DefaultTbClusterService implements TbClusterService { } private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - TransportProtos.CalculatedFieldEntityUpdateMsgProto.Builder builder = TransportProtos.CalculatedFieldEntityUpdateMsgProto.newBuilder(); + CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); builder.setEntityType(entityId.getEntityType().name()); @@ -846,9 +868,22 @@ public class DefaultTbClusterService implements TbClusterService { builder.setAdded(added); builder.setUpdated(updated); builder.setDeleted(deleted); - TransportProtos.CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + CalculatedFieldEntityUpdateMsgProto msg = builder.build(); + + broadcastEntityUpdateEvent(msg); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + } - pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); + private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { + TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); + for (String serviceId : tbCalculatedFieldServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); + toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); + toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 5b1e5d7d79..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,16 +178,12 @@ public abstract class AbstractConsumerService> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toCalculatedFields; + private TbQueueProducer> toCalculatedFields; private TbQueueProducer> toCalculatedFieldNotifications; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { From d3278f05bb4658bc109b71b927afd2b78e2bea88 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:34:35 +0200 Subject: [PATCH 098/438] WIP: Cluster mode refactoring --- .../server/actors/ActorSystemContext.java | 6 + .../server/actors/app/AppActor.java | 25 ++ .../CalculatedFieldEntityActor.java | 65 ++++ .../CalculatedFieldEntityActorCreator.java | 49 +++ ...CalculatedFieldEntityMessageProcessor.java | 199 +++++++++++ .../CalculatedFieldLinkedTelemetryMsg.java | 40 +++ .../CalculatedFieldManagerActor.java | 84 +++++ .../CalculatedFieldManagerActorCreator.java | 46 +++ ...alculatedFieldManagerMessageProcessor.java | 149 +++++++++ .../CalculatedFieldStateRestoreMsg.java | 40 +++ .../CalculatedFieldTelemetryMsg.java | 39 +++ .../EntityCalculatedFieldTelemetryMsg.java | 55 +++ .../calculatedField/MultipleTbCallback.java | 49 +++ .../actors/service/DefaultActorService.java | 11 + .../server/actors/tenant/TenantActor.java | 29 +- .../cf/CalculatedFieldExecutionService.java | 23 +- .../cf/CalculatedFieldInitService.java | 19 ++ .../cf/DefaultCalculatedFieldCache.java | 2 + ...efaultCalculatedFieldExecutionService.java | 313 ++++++++---------- .../cf/DefaultCalculatedFieldInitService.java | 64 ++++ .../cf/ctx/CalculatedFieldEntityCtxId.java | 3 +- .../cf/ctx/CalculatedFieldStateService.java | 9 +- .../ctx/state/BaseCalculatedFieldState.java | 5 + .../cf/ctx/state/CalculatedFieldCtx.java | 1 + .../cf/ctx/state/CalculatedFieldState.java | 1 + .../cf/ctx/state/RocksDBStateService.java | 14 +- .../ctx/state/SingleValueArgumentEntry.java | 16 + ...faultTbCalculatedFieldConsumerService.java | 48 +-- .../DefaultTelemetrySubscriptionService.java | 4 +- .../src/main/resources/thingsboard.yml | 2 + .../resources/application-test.properties | 2 + .../TbCalculatedFieldEntityActorId.java | 56 ++++ .../cf/configuration/ReferencedEntityKey.java | 2 + .../server/common/msg/MsgType.java | 11 +- .../msg/ToCalculatedFieldSystemMsg.java | 27 ++ .../common/msg/cf/CalculatedFieldInitMsg.java | 34 ++ .../msg/cf/CalculatedFieldLinkInitMsg.java | 34 ++ .../server/queue/util/AfterStartUp.java | 4 + .../resources/application-test.properties | 2 + 39 files changed, 1362 insertions(+), 220 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java create mode 100644 common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e6f541090c..09488bfe4e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -104,6 +104,7 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -507,6 +508,11 @@ public class ActorSystemContext { @Getter private EntityService entityService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 622812b327..9124dc0a9a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.TenantAwareMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; @@ -111,6 +112,17 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } @@ -175,6 +187,19 @@ public class AppActor extends ContextAwareActor { } } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { + msg.getCallback().onSuccess(); + }); + } + + private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java new file mode 100644 index 0000000000..43f057ad01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; + +@Slf4j +public class CalculatedFieldEntityActor extends ContextAwareActor { + + private final CalculatedFieldEntityMessageProcessor processor; + + CalculatedFieldEntityActor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.processor = new CalculatedFieldEntityMessageProcessor(systemContext, tenantId, entityId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}][{}] Starting CF entity actor.", processor.tenantId, processor.entityId); + try { + processor.init(ctx); + log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); + } catch (Exception e) { + log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); + throw new TbActorException("Failed to initialize device actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case CF_STATE_RESTORE_MSG: + processor.process((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_ENTITY_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldTelemetryMsg) msg); + break; + default: + return false; + } + return true; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java new file mode 100644 index 0000000000..c5f8ecf046 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.device.DeviceActor; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + private final EntityId entityId; + + public CalculatedFieldEntityActorCreator(ActorSystemContext context, TenantId tenantId, EntityId entityId) { + super(context); + this.tenantId = tenantId; + this.entityId = entityId; + } + + @Override + public TbActorId createActorId() { + return new TbEntityActorId(entityId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldEntityActor(context, tenantId, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java new file mode 100644 index 0000000000..6bb6256563 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -0,0 +1,199 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldResult; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareMsgProcessor { + // (1 for result persistence + 1 for the state persistence ) + public static final int CALLBACKS_PER_CF = 2; + + final TenantId tenantId; + final EntityId entityId; + final CalculatedFieldExecutionService cfService; + + TbActorCtx ctx; + Map states = new HashMap<>(); + + CalculatedFieldEntityMessageProcessor(ActorSystemContext systemContext, TenantId tenantId, EntityId entityId) { + super(systemContext); + this.tenantId = tenantId; + this.entityId = entityId; + this.cfService = systemContext.getCalculatedFieldExecutionService(); + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void process(EntityCalculatedFieldTelemetryMsg msg) { + var proto = msg.getProto(); + var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); + MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); + List cfIdList = getCalculatedFieldIds(proto); + Set cfIdSet = new HashSet<>(cfIdList); + for (var ctx : msg.getEntityIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + for (var ctx : msg.getProfileIdFields()) { + process(ctx, proto, cfIdSet, cfIdList, callback); + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processTelemetry(ctx, proto, cfIdList, callback); + } else if (proto.getAttrDataCount() > 0) { + processAttributes(ctx, proto, cfIdList, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + @SneakyThrows + private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + } + + @SneakyThrows + private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + } + + private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, + Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + if (newArgValues.isEmpty()) { + callback.onSuccess(CALLBACKS_PER_CF); + } + CalculatedFieldState state = getOrInitState(ctx); + if (state.updateState(newArgValues)) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + + @SneakyThrows + private CalculatedFieldState getOrInitState(CalculatedFieldCtx ctx) { + CalculatedFieldState state = states.get(ctx.getCfId()); + if (state != null) { + return state; + } else { + ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } + return state; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (TsKvProto item : data) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_ROLLING, null); + argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + Map arguments = new HashMap<>(); + var argNames = ctx.getMainEntityArguments(); + for (AttributeValueProto item : attrDataList) { + ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); + String argName = argNames.get(key); + if (argName != null) { + arguments.put(argName, new SingleValueArgumentEntry(item)); + } + } + return arguments; + } + + private static List getCalculatedFieldIds(CalculatedFieldTelemetryMsgProto proto) { + List cfIds = new LinkedList<>(); + for (var cfId : proto.getPreviousCalculatedFieldsList()) { + cfIds.add(new CalculatedFieldId(new UUID(cfId.getCalculatedFieldIdMSB(), cfId.getCalculatedFieldIdLSB()))); + } + return cfIds; + } + + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..0c352f5072 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldLinkedTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java new file mode 100644 index 0000000000..87292d3206 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorException; +import org.thingsboard.server.actors.service.ContextAwareActor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; + +/** + * Created by ashvayka on 15.03.18. + */ +@Slf4j +public class CalculatedFieldManagerActor extends ContextAwareActor { + + private final CalculatedFieldManagerMessageProcessor processor; + + public CalculatedFieldManagerActor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.processor = new CalculatedFieldManagerMessageProcessor(systemContext, tenantId); + } + + @Override + public void init(TbActorCtx ctx) throws TbActorException { + super.init(ctx); + log.debug("[{}] Starting CF manager actor.", processor.tenantId); + try { + processor.init(ctx); + log.debug("[{}] CF manager actor started.", processor.tenantId); + } catch (Exception e) { + log.warn("[{}] Unknown failure", processor.tenantId, e); + throw new TbActorException("Failed to initialize manager actor", e); + } + } + + @Override + protected boolean doProcess(TbActorMsg msg) { + switch (msg.getMsgType()) { + case PARTITION_CHANGE_MSG: + ctx.broadcastToChildren(msg, true); // TODO + break; + case CF_INIT_MSG: + processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); + break; + case CF_LINK_INIT_MSG: + processor.onLinkInitMsg((CalculatedFieldLinkInitMsg) msg); + break; + case CF_STATE_RESTORE_MSG: + processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); + break; + case CF_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + case CF_TELEMETRY_MSG: + processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); + break; + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: +// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + break; + default: + return false; + } + return true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java new file mode 100644 index 0000000000..2a53089bed --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActorCreator.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActor; +import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbEntityActorId; +import org.thingsboard.server.actors.TbStringActorId; +import org.thingsboard.server.actors.service.ContextBasedCreator; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public class CalculatedFieldManagerActorCreator extends ContextBasedCreator { + + private final TenantId tenantId; + + public CalculatedFieldManagerActorCreator(ActorSystemContext context, TenantId tenantId) { + super(context); + this.tenantId = tenantId; + } + + @Override + public TbActorId createActorId() { + return new TbStringActorId("CFM|" + tenantId); + } + + @Override + public TbActor createActor() { + return new CalculatedFieldManagerActor(context, tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java new file mode 100644 index 0000000000..4bc0589e95 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -0,0 +1,149 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.TbActorCtx; +import org.thingsboard.server.actors.TbActorRef; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; +import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; + + +/** + * @author Andrew Shvayka + */ +@Slf4j +public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { + + private final Map calculatedFields = new HashMap<>(); + private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + + protected TbActorCtx ctx; + final TenantId tenantId; + + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { + super(systemContext); + this.assetProfileCache = systemContext.getAssetProfileCache(); + this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.tenantId = tenantId; + } + + void init(TbActorCtx ctx) { + this.ctx = ctx; + } + + public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + var cf = msg.getCf(); + calculatedFields.put(cf.getId(), cf); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) + .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + msg.getCallback().onSuccess(); + } + + public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + var link = msg.getLink(); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + msg.getCallback().onSuccess(); + } + + public void onStateRestoreMsg(CalculatedFieldStateRestoreMsg msg) { + if (calculatedFields.containsKey(msg.getId().cfId())) { + getOrCreateActor(msg.getId().entityId()).tell(msg); + } else { + // TODO: remove state from storage + } + } + + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + // process all cfs related to entity, or it's profile; + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + //TODO: Transfer only 'part' of the original callback. + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + // process all links (if any); + var links = getCalculatedFieldLinksByEntityId(entityId); + } + + private List getCalculatedFieldsByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFields.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private List getCalculatedFieldLinksByEntityId(EntityId entityId) { + if (entityId == null) { + return Collections.emptyList(); + } + var result = entityIdCalculatedFieldLinks.get(entityId); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + protected TbActorRef getOrCreateActor(EntityId entityId) { + return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), + () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, + () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), + () -> true); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java new file mode 100644 index 0000000000..7c0e67f0ec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldStateRestoreMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + +@Data +public class CalculatedFieldStateRestoreMsg implements ToCalculatedFieldSystemMsg { + + private final CalculatedFieldEntityCtxId id; + private final CalculatedFieldState state; + + @Override + public MsgType getMsgType() { + return MsgType.CF_STATE_RESTORE_MSG; + } + + @Override + public TenantId getTenantId() { + return id.tenantId(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..5cce83b00b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldTelemetryMsg.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; + +@Data +public class CalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final TbCallback callback; + + + @Override + public MsgType getMsgType() { + return MsgType.CF_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java new file mode 100644 index 0000000000..8465bd8fcc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldTelemetryMsg.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final List entityIdFields; + private final List profileIdFields; + private final TbCallback callback; + + public EntityCalculatedFieldTelemetryMsg(CalculatedFieldTelemetryMsg msg, + List entityIdFields, + List profileIdFields, + TbCallback callback) { + this.tenantId = msg.getTenantId(); + this.entityId = msg.getEntityId(); + this.proto = msg.getProto(); + this.entityIdFields = entityIdFields; + this.profileIdFields = profileIdFields; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java new file mode 100644 index 0000000000..18e700f38c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import org.thingsboard.server.common.msg.queue.TbCallback; + +import java.util.concurrent.atomic.AtomicInteger; + +public class MultipleTbCallback implements TbCallback { + + private final AtomicInteger counter; + private final TbCallback callback; + + public MultipleTbCallback(int count, TbCallback callback) { + this.counter = new AtomicInteger(count); + this.callback = callback; + } + + @Override + public void onSuccess() { + if (counter.decrementAndGet() <= 0) { + callback.onSuccess(); + } + } + + public void onSuccess(int number) { + if (counter.addAndGet(-number) <= 0) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 492940a274..7e0d1b4447 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -49,6 +49,8 @@ public class DefaultActorService extends TbApplicationEventListener deletedDevices; + private TbActorRef cfActor; private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); @@ -95,6 +98,11 @@ public class TenantActor extends RuleChainManagerActor { } else { log.info("[{}] Skip init of the rule chains due to API limits", tenantId); } + //TODO: IM - extend API usage to have CF Exec Enabled? Not in 4.0; + cfActor = ctx.getOrCreateChildActor(new TbStringActorId("CFM|" + tenantId), + () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, + () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), + () -> true); } catch (Exception e) { log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); cantFindTenant = true; @@ -159,12 +167,31 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_INIT_MSG: + case CF_LINK_INIT_MSG: + case CF_STATE_RESTORE_MSG: + case CF_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); + break; + case CF_TELEMETRY_MSG: + case CF_LINKED_TELEMETRY_MSG: + case CF_ENTITY_UPDATE_MSG: + onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); + break; default: return false; } return true; } + private void onToCalculatedFieldSystemActorMsg(ToCalculatedFieldSystemMsg msg, boolean priority) { + if (priority) { + cfActor.tellWithHighPriority(msg); + } else { + cfActor.tell(msg); + } + } + private boolean isMyPartition(EntityId entityId) { return systemContext.resolve(ServiceType.TB_CORE, tenantId, entityId).isMyPartition(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 62e234943b..d38c1c58ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -15,14 +15,22 @@ */ package org.thingsboard.server.service.cf; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; @@ -30,16 +38,17 @@ public interface CalculatedFieldExecutionService { /** * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * @param request - telemetry save request; - * @param request - telemetry save result; + * + * @param request - telemetry save request; + * @param callback */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result); + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - void pushRequestToQueue(AttributesSaveRequest request, List result); + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback); + void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback); + ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); @@ -53,4 +62,6 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java new file mode 100644 index 0000000000..25f92e797f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldInitService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +public interface CalculatedFieldInitService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 4278e74845..b0c9f6adde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f92b9819ed..bb8ba8bb63 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -42,7 +41,6 @@ import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; -import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -63,7 +61,6 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -108,12 +105,13 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.Supplier; @@ -167,7 +165,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -192,22 +189,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result) { + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { var tenantId = request.getTenantId(); var entityId = request.getEntityId(); checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), request.getCallback()); + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); } private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, @@ -241,77 +238,84 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryMsg(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { - - callback.onSuccess(); + public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { + Map> argFutures = new HashMap<>(); + for (var entry : ctx.getArguments().entrySet()) { + var argEntityId = entry.getValue().getRefEntityId() != null ? entry.getValue().getRefEntityId() : entityId; + var argValueFuture = fetchKvEntry(ctx.getTenantId(), argEntityId, entry.getValue()); + argFutures.put(entry.getKey(), argValueFuture); + } + return Futures.whenAllComplete(argFutures.values()).call(() -> { + var result = createStateByType(ctx.getCfType()); + result.updateState(argFutures.entrySet().stream() + .collect(Collectors.toMap( + Entry::getKey, // Keep the key as is + entry -> { + try { + // Resolve the future to get the value + return entry.getValue().get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Error getting future result for key: " + entry.getKey(), e); + } + } + ))); + return result; + }, calculatedFieldCallbackExecutor); } @Override - public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - - callback.onSuccess(); + public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(stateId, state, callback); } @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - Map> tpiTargetEntityMap = new HashMap<>(); - - for (CalculatedField cf : cfs) { - - Consumer resolvePartition = entityId -> { - TopicPartitionInfo tpi; - try { - tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); - } - } catch (Exception e) { - log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", - entityId, cf.getTenantId(), e.getMessage()); - } - }; - - EntityId cfEntityId = cf.getEntityId(); - if (isProfileEntity(cfEntityId)) { - calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); - } else { - resolvePartition.accept(cfEntityId); - } - } - - for (var entry : tpiTargetEntityMap.entrySet()) { - for (List partition : Lists.partition(entry.getValue(), 1000)) { - log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); - var future = calculatedFieldExecutor.submit(() -> { - try { - for (CalculatedFieldEntityCtxId ctxId : partition) { - restoreState(ctxId.cfId(), ctxId.entityId()); - } - } catch (Throwable t) { - log.error("Unexpected exception while restoring CalculatedField states", t); - throw t; - } - }); - result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); - } - } +// PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); +// Map> tpiTargetEntityMap = new HashMap<>(); +// +// for (CalculatedField cf : cfs) { +// +// Consumer resolvePartition = entityId -> { +// TopicPartitionInfo tpi; +// try { +// tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); +// if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { +// tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); +// } +// } catch (Exception e) { +// log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", +// entityId, cf.getTenantId(), e.getMessage()); +// } +// }; +// +// EntityId cfEntityId = cf.getEntityId(); +// if (isProfileEntity(cfEntityId)) { +// calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); +// } else { +// resolvePartition.accept(cfEntityId); +// } +// } +// +// for (var entry : tpiTargetEntityMap.entrySet()) { +// for (List partition : Lists.partition(entry.getValue(), 1000)) { +// log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); +// var future = calculatedFieldExecutor.submit(() -> { +// try { +// for (CalculatedFieldEntityCtxId ctxId : partition) { +// restoreState(ctxId.cfId(), ctxId.entityId()); +// } +// } catch (Throwable t) { +// log.error("Unexpected exception while restoring CalculatedField states", t); +// throw t; +// } +// }); +// result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); +// } +// } return result; } - private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - - if (restoredCtx != null) { - states.put(ctxId, restoredCtx); - log.info("Restored state for CalculatedField [{}]", calculatedFieldId); - } else { - log.warn("No state found for CalculatedField [{}], entity [{}].", calculatedFieldId, entityId); - } - } - @Override protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { cleanupEntity(entityId); @@ -491,7 +495,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); } } @@ -525,7 +529,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); +// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -569,9 +573,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - stateService.removeState(ctxId); } private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { @@ -601,7 +602,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { @Override public void onSuccess(List results) { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); +// updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); callback.onSuccess(); } @@ -613,96 +614,83 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); - Map argumentsMap = new HashMap<>(argumentValues); - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - - CompletableFuture updateFuture = new CompletableFuture<>(); - - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - stateService.persistState(entityCtxId, calculatedFieldEntityCtx); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); - } - updateFuture.complete(null); - }; - - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); - - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } - - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } - - return calculatedFieldEntityCtx; - }); - } - - private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { - ListenableFuture resultFuture = state.performCalculation(calculatedFieldCtx); - Futures.addCallback(resultFuture, new FutureCallback<>() { - @Override - public void onSuccess(CalculatedFieldResult result) { - if (result != null) { - pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, previousCalculatedFieldIds); - } - } +// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { +// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); +// Map argumentsMap = new HashMap<>(argumentValues); +// +// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); +// +// states.compute(entityCtxId, (ctxId, ctx) -> { +// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); +// +// CompletableFuture updateFuture = new CompletableFuture<>(); +// +// Consumer performUpdateState = (state) -> { +// if (state.updateState(argumentsMap)) { +// calculatedFieldEntityCtx.setState(state); +// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); +// Map arguments = state.getArguments(); +// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && +// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); +// if (allArgsPresent) { +// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); +// } +// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); +// } +// updateFuture.complete(null); +// }; +// +// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); +// +// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); +// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() +// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); +// +// if (!allKeysPresent || requiresTsRollingUpdate) { +// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() +// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) +// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +// +// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) +// .addListener(() -> performUpdateState.accept(state), +// calculatedFieldCallbackExecutor); +// } else { +// performUpdateState.accept(state); +// } +// +// try { +// updateFuture.join(); +// } catch (Exception e) { +// log.trace("Failed to update state for ctxId [{}].", ctxId, e); +// throw new RuntimeException("Failed to update or initialize state.", e); +// } +// +// return calculatedFieldEntityCtx; +// }); +// } - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t); - } - }, MoreExecutors.directExecutor()); - } - - private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List previousCalculatedFieldIds) { + @Override + public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { OutputType type = calculatedFieldResult.getType(); TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST; TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY; ObjectNode payload = createJsonPayload(calculatedFieldResult); - if (previousCalculatedFieldIds != null && previousCalculatedFieldIds.contains(calculatedFieldId)) { - throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop."); - } - List calculatedFieldIds = previousCalculatedFieldIds != null - ? new ArrayList<>(previousCalculatedFieldIds) - : new ArrayList<>(); - calculatedFieldIds.add(calculatedFieldId); - TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).previousCalculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); - clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null); - log.info("Pushed message to rule engine: originatorId=[{}]", originatorId); + TbMsg msg = TbMsg.newMsg().type(msgType).originator(entityId).previousCalculatedFieldIds(cfIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build(); + clusterService.pushMsgToRuleEngine(tenantId, entityId, msg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + log.trace("[{}][{}] Pushed message to rule engine: {} ", tenantId, entityId, msg); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + }); } catch (Exception e) { - log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e); + log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); } } @@ -781,15 +769,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return new StringDataEntry(key, defaultValue); } - private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); - - if (state == null) { - return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); - } - return state; - } - private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { ObjectNode payload = JacksonUtil.newObjectNode(); Map resultMap = calculatedFieldResult.getResultMap(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java new file mode 100644 index 0000000000..87dda7013f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +@Service +@TbRuleEngineComponent +@RequiredArgsConstructor +public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { + + private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldStateService stateService; + private final ActorSystemContext actorSystemContext; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + + @AfterStartUp(order = AfterStartUp.CF_INIT_SERVICE) + public void initCalculatedFieldDefinitions() { + PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); + cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); + PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); + cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); + //TODO: combine with the DefaultCalculatedFieldCache. + + } + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 5fb90a3e46..6ce0a11ade 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; -public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { +public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 8bc5756f4e..2bf6b7e0f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -15,15 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; + import java.util.Map; public interface CalculatedFieldStateService { - Map restoreStates(); + Map restoreStates(); - CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index bc9a421e47..ca243e25b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -66,4 +66,9 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { return stateUpdated; } + @Override + public boolean isReady() { + //TODO: IM + return true; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e4abbee4cd..1a0a16254a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 3c4a680df1..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,4 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index db8950804c..aa82f2fc57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -36,24 +37,25 @@ public class RocksDBStateService implements CalculatedFieldStateService { private final RocksDBService rocksDBService; @Override - public Map restoreStates() { + public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) )); } @Override - public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) .orElse(null); } @Override - public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { - rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ + rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + callback.onSuccess(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d5080e90f..d5f7b1aee6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,6 +21,10 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @Data @NoArgsConstructor @@ -34,6 +38,18 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private Long version; + public SingleValueArgumentEntry(TsKvProto entry) { + this.ts = entry.getTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + + public SingleValueArgumentEntry(AttributeValueProto entry) { + this.ts = entry.getLastUpdateTs(); + this.version = entry.getVersion(); + this.value = ProtoUtils.fromProto(entry).getValue(); + } + public SingleValueArgumentEntry(KvEntry entry) { if (entry instanceof TsKvEntry tsKvEntry) { this.ts = tsKvEntry.getTs(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index e550a1154e..fb1d6b19ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,8 +21,6 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; -import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -30,23 +28,20 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; -import org.thingsboard.server.common.msg.MsgType; -import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -166,10 +161,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer pendingMsgHolder.setMsg(toCfMsg); if (toCfMsg.hasTelemetryMsg()) { log.trace("[{}] Forwarding regular telemetry message for processing {}", id, toCfMsg.getTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getTelemetryMsg(), callback); } else if (toCfMsg.hasLinkedTelemetryMsg()) { log.trace("[{}] Forwarding linked telemetry message for processing {}", id, toCfMsg.getLinkedTelemetryMsg()); - forwardToCalculatedFieldService(toCfMsg.getLinkedTelemetryMsg(), callback); + forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -219,9 +214,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - forwardToCalculatedFieldService(toCfNotification.getComponentLifecycle(), callback); + forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); } else if (toCfNotification.hasEntityUpdateMsg()) { - forwardToCalculatedFieldService(toCfNotification.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); } callback.onSuccess(); } @@ -249,33 +244,20 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer // } // - private void forwardToCalculatedFieldService(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { - var msg = linkedMsg.getMsg(); + private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onLinkedTelemetryMsg(linkedMsg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldTelemetryMsg(tenantId, entityId, msg, callback)); } - private void forwardToCalculatedFieldService(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(CalculatedFieldLinkedTelemetryMsgProto linkedMsg, TbCallback callback) { + var msg = linkedMsg.getMsg(); var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onTelemetryMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); + var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.ComponentLifecycleMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); @@ -287,7 +269,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer }); } - private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { + private void forwardToActorSystem(TransportProtos.CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index df18640359..3f7f9c0b7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result); + calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d62aac5e6c..30a3e51d8c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -441,6 +441,8 @@ actors: device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors edge_dispatcher_pool_size: "${ACTORS_SYSTEM_EDGE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for edge actors + cfm_dispatcher_pool_size: "${ACTORS_SYSTEM_CFM_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for CalculatedField manager actors + cfe_dispatcher_pool_size: "${ACTORS_SYSTEM_CFE_DISPATCHER_POOL_SIZE:8}" # Thread pool size for actor system dispatcher that process messages for CalculatedField entity actors tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 9951caa876..a1315cc8bd 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -35,6 +35,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java new file mode 100644 index 0000000000..66522cd08d --- /dev/null +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbCalculatedFieldEntityActorId.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.Objects; + +public class TbCalculatedFieldEntityActorId implements TbActorId { + + @Getter + private final EntityId entityId; + + public TbCalculatedFieldEntityActorId(EntityId entityId) { + this.entityId = entityId; + } + + @Override + public String toString() { + return entityId.getEntityType() + "|" + entityId.getId(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TbCalculatedFieldEntityActorId that = (TbCalculatedFieldEntityActorId) o; + return entityId.equals(that.entityId); + } + + @Override + public int hashCode() { + // Magic number to ensure that the hash does not match with the hash of other actor id - (TbEntityActorId) + return 42 + Objects.hash(entityId); + } + + @Override + public EntityType getEntityType() { + return entityId.getEntityType(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b49495d959..b4bcc77a17 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -27,4 +27,6 @@ public class ReferencedEntityKey { private ArgumentType type; private AttributeScope scope; + + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index b05bb75032..91419bee9c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -133,7 +133,16 @@ public enum MsgType { * Messages that are sent to and from edge session to start edge synchronization process */ EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, - EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG; + EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + + CF_INIT_MSG, // Sent to init particular calculated field; + CF_LINK_INIT_MSG, // Sent to init particular calculated field; + CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; + CF_TELEMETRY_MSG, + CF_ENTITY_TELEMETRY_MSG, + CF_LINKED_TELEMETRY_MSG, + CF_UPDATE_MSG, + CF_ENTITY_UPDATE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java new file mode 100644 index 0000000000..2da959ec7d --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/ToCalculatedFieldSystemMsg.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg; + +import org.thingsboard.server.common.msg.aware.TenantAwareMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +public interface ToCalculatedFieldSystemMsg extends TenantAwareMsg { + + default TbCallback getCallback() { + return TbCallback.EMPTY; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java new file mode 100644 index 0000000000..4d9f4ce414 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedField cf; + + @Override + public MsgType getMsgType() { + return MsgType.CF_INIT_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java new file mode 100644 index 0000000000..7f582e6ff8 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldLinkInitMsg.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldLinkInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldLink link; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINK_INIT_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 53ff839c49..70ae853e1e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,6 +38,10 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; + int CF_INIT_SERVICE = 10; + int CF_STATE_RESTORE_SERVICE = 11; + int CF_CONSUMER_SERVICE = 12; + int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; int AFTER_TRANSPORT_SERVICE = Integer.MAX_VALUE - 999; diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 1e11c77f9a..a1bb335ad0 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -145,6 +145,8 @@ sql.events.batch_threads=2 actors.system.tenant_dispatcher_pool_size=4 actors.system.device_dispatcher_pool_size=8 actors.system.rule_dispatcher_pool_size=12 +actors.system.cfm_dispatcher_pool_size=2 +actors.system.cfe_dispatcher_pool_size=2 transport.sessions.report_timeout=10000 queue.transport_api.request_poll_interval=5 queue.transport_api.response_poll_interval=5 From 86b5378d592de789afff096e387bb328389c48a7 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 27 Jan 2025 15:16:50 +0200 Subject: [PATCH 099/438] EDQS (#3196) * Experiments with CSV * CSV Loader v1 * EDQ tests * Volatile variables instead of final * Improvements * updated loader with new entities * Fix double memory usage issue * Basic data structures and load * Minor improvements * Snappy + Large String reuse * added EntityFields classes for each entity * Basic implementation * Minor improvements to KeyFilters * implemented RepositoryUtils.checkKeyFilters * Generic query implementation * New structure * Refactoring and few processors implementation * extended DeviceData with shared/client attributes and device profile * Minor refactoring of attribute scopes * DeviceTypeFilter support * Strong types of fields for each entity data class * DeviceType and AssetType filters * EntityView and Edge queries * Relations Query * Relation Query Implementation * Update EDQS module version * Sync with EDQS via Kafka * EDQS: major refactoring * EDQS API requests via Kafka * EDQS: full sync with the database * Refactoring for EDQS sync * EDQS: major refactoring and new features * EDQS refactoring, count query support, fix tests * EDQS: refactoring for query processors * Fix EDQS pom version * Cleanup edqs.yml * EDQS: tenant partitioning strategy; refactoring * EDQS: latest events queue * EDQS: support for monolith setup; RocksDB; other improvements * EDQS: merge sync and events topics, introduce state topic * EDQS: dynamic repartitioning * implemented entity data query filters for edqs * EdqsEntityQueryControllerTest - use in-memory queue * edqs-filter fixes, added test * EDQS: blob entity support * EdqsEntityQueryControllerTest - use in-memory queue * Use DummyEdqsService when disabled * Fixes for EDQS * Refactoring for EDQS tests * Fix edqs requests partitioning * EDQS: Fix for attributes handling * Fix attributes saving in EntityServiceTest * EDQS: refactoring, fixes * Minor refactoring for query processor * added ownerName/ownerType support * fixed relation query processor * fixed EntityServiceTest * refactoring * added support for parentId for relation query result * Get rid of EntityNameFetcher * Add fixme for relation query processor * db restore with select all edqs fields * fixed entity deletion * fixed FieldUtils with new EntityFields * dao method renamed * EDQS: instance groups with same partitions; automatic sync; multiple fixes * Refactoring for EDQS sync * EDQS: refactoring * Fix startup with Kafka * fixed EntityQueryControllerTest * fixed EdqsEntityServiceTest * Separate queue admin for EDQS request template * Implement new EDQS partitioning strategy * EDQS: multiple fixes and refactoring * Add mock EdqsRocksDb beans to tests * added edqs stats for inmemory/grafana * fixed filter tests * Update todos * Refactoring for QueueConfig * Improvements and refactoring for EDQS consumers * implemented TODOs * test fixes * Consume state topic up to end offsets * edqs stats refactoring * EDQS: cleanup on partitions removal; refactoring * EDQS: minor refactoring * EDQS: remove CSV loader --------- Co-authored-by: Andrii Shvaika Co-authored-by: dashevchenko --- application/pom.xml | 4 + .../controller/EntityQueryController.java | 11 + .../edge/EdgeEventSourcingListener.java | 5 + .../service/edqs/DefaultEdqsService.java | 340 ++++++++++ .../server/service/edqs/EdqsDataLoader.java | 539 ++++++++++++++++ .../server/service/edqs/EdqsListener.java | 61 ++ .../server/service/edqs/EdqsSyncService.java | 275 ++++++++ .../service/edqs/KafkaEdqsSyncService.java | 47 ++ .../service/edqs/LocalEdqsSyncService.java | 35 + .../entitiy/EntityStateSourcingListener.java | 5 + .../queue/DefaultTbCoreConsumerService.java | 27 +- .../queue/DefaultTbEdgeConsumerService.java | 16 +- .../DefaultTbRuleEngineConsumerService.java | 2 +- .../TbRuleEngineQueueConsumerManager.java | 5 +- .../state/DefaultDeviceStateService.java | 31 +- .../sync/tenant/TenantExportService.java | 224 +++++++ .../transport/TbCoreTransportApiService.java | 3 +- .../service/ws/DefaultWebSocketService.java | 1 + .../src/main/resources/thingsboard.yml | 23 + .../EdqsEntityQueryControllerTest.java | 67 ++ .../controller/EntityQueryControllerTest.java | 167 +++-- .../discovery/HashPartitionServiceTest.java | 16 +- .../entitiy/EdqsEntityServiceTest.java | 72 +++ .../service/entitiy}/EntityServiceTest.java | 609 ++++++++++-------- .../TbRuleEngineQueueConsumerManagerTest.java | 11 +- .../state/DefaultDeviceStateServiceTest.java | 2 +- .../resources/application-test.properties | 5 +- .../src/test/resources/logback-test.xml | 2 +- .../server/queue/TbQueueRequestTemplate.java | 2 + .../server/queue/TbQueueResponseTemplate.java | 10 +- .../server/common/data/ObjectType.java | 102 +++ .../server/common/data/alarm/AlarmType.java | 27 + .../server/common/data/edqs/AttributeKv.java | 67 ++ .../server/common/data/edqs/EdqsEvent.java | 34 + .../common/data/edqs/EdqsEventType.java | 21 + .../server/common/data/edqs/EdqsObject.java | 28 + .../common/data/edqs/EdqsSyncRequest.java | 24 + .../server/common/data/edqs/Entity.java | 62 ++ .../server/common/data/edqs/LatestTsKv.java | 62 ++ .../common/data/edqs/ToCoreEdqsMsg.java | 32 + .../common/data/edqs/ToCoreEdqsRequest.java | 38 ++ .../edqs/fields/AbstractEntityFields.java | 65 ++ .../data/edqs/fields/ApiUsageStateFields.java | 58 ++ .../common/data/edqs/fields/AssetFields.java | 58 ++ .../data/edqs/fields/AssetProfileFields.java | 35 + .../data/edqs/fields/CustomerFields.java | 55 ++ .../data/edqs/fields/DashboardFields.java | 33 + .../common/data/edqs/fields/DeviceFields.java | 58 ++ .../data/edqs/fields/DeviceProfileFields.java | 38 ++ .../common/data/edqs/fields/EdgeFields.java | 43 ++ .../common/data/edqs/fields/EntityFields.java | 171 +++++ .../data/edqs/fields/EntityIdFields.java | 36 ++ .../data/edqs/fields/EntityViewFields.java | 30 + .../common/data/edqs/fields/FieldsUtil.java | 298 +++++++++ .../data/edqs/fields/GenericFields.java | 41 ++ .../data/edqs/fields/ProfileAwareFields.java | 26 + .../data/edqs/fields/QueueStatsFields.java | 42 ++ .../data/edqs/fields/RuleChainFields.java | 38 ++ .../data/edqs/fields/RuleNodeFields.java | 43 ++ .../common/data/edqs/fields/TenantFields.java | 62 ++ .../data/edqs/fields/TenantProfileFields.java | 36 ++ .../common/data/edqs/fields/UserFields.java | 48 ++ .../data/edqs/fields/WidgetTypeFields.java | 30 + .../data/edqs/fields/WidgetsBundleFields.java | 30 + .../common/data/edqs/query/EdqsRequest.java | 38 ++ .../common/data/edqs/query/EdqsResponse.java | 35 + .../common/data/edqs/query/QueryResult.java | 41 ++ .../common/data/id/UserAuthSettingsId.java | 6 +- .../data/page/BasePageDataIterable.java | 9 +- .../common/data/page/PageDataIterable.java | 5 + .../common/data/permission/QueryContext.java | 14 +- .../common/data/query/DynamicValue.java | 2 - .../common/data/query/EntityCountQuery.java | 2 + .../server/common/data/query/EntityData.java | 15 +- .../server/common/data/queue/QueueConfig.java | 12 + .../server/common/data/queue/QueueStats.java | 8 +- .../common/data/relation/EntityRelation.java | 15 +- .../common/data/util/CollectionsUtil.java | 8 + common/edqs/pom.xml | 97 +++ .../server/edqs/data/ApiUsageStateData.java | 46 ++ .../server/edqs/data/AssetData.java | 36 ++ .../server/edqs/data/BaseEntityData.java | 180 ++++++ .../server/edqs/data/CustomerData.java | 62 ++ .../server/edqs/data/DeviceData.java | 86 +++ .../server/edqs/data/EntityData.java | 65 ++ .../server/edqs/data/EntityGroupData.java | 58 ++ .../server/edqs/data/EntityProfileData.java | 39 ++ .../server/edqs/data/GenericData.java | 38 ++ .../server/edqs/data/ProfileAwareData.java | 28 + .../server/edqs/data/RelationData.java | 26 + .../server/edqs/data/RelationInfo.java | 26 + .../server/edqs/data/RelationsRepo.java | 62 ++ .../server/edqs/data/TenantData.java | 34 + .../edqs/data/dp/AbstractDataPoint.java | 56 ++ .../server/edqs/data/dp/BoolDataPoint.java | 46 ++ .../edqs/data/dp/CompressedJsonDataPoint.java | 31 + .../data/dp/CompressedStringDataPoint.java | 64 ++ .../server/edqs/data/dp/DataPoint.java | 40 ++ .../server/edqs/data/dp/DoubleDataPoint.java | 46 ++ .../server/edqs/data/dp/JsonDataPoint.java | 47 ++ .../server/edqs/data/dp/LongDataPoint.java | 50 ++ .../server/edqs/data/dp/StringDataPoint.java | 51 ++ .../server/edqs/load/TenantRepoLoader.java | 144 +++++ .../server/edqs/processor/EdqsConverter.java | 183 ++++++ .../server/edqs/processor/EdqsProcessor.java | 266 ++++++++ .../server/edqs/processor/EdqsProducer.java | 83 +++ .../server/edqs/query/DataKey.java | 22 + .../server/edqs/query/EdqsCountQuery.java | 30 + .../server/edqs/query/EdqsDataQuery.java | 59 ++ .../server/edqs/query/EdqsFilter.java | 23 + .../server/edqs/query/EdqsQuery.java | 30 + .../server/edqs/query/SortableEntityData.java | 40 ++ .../AbstractEntityGroupQueryProcessor.java | 75 +++ ...stractEntityProfileNameQueryProcessor.java | 52 ++ .../AbstractEntityProfileQueryProcessor.java | 62 ++ .../AbstractEntitySearchQueryProcessor.java | 66 ++ .../processor/AbstractQueryProcessor.java | 129 ++++ .../AbstractRelationQueryProcessor.java | 260 ++++++++ .../AbstractSimpleQueryProcessor.java | 99 +++ ...bstractSingleEntityTypeQueryProcessor.java | 172 +++++ .../ApiUsageStateQueryProcessor.java | 88 +++ .../processor/AssetSearchQueryProcessor.java | 59 ++ .../processor/AssetTypeQueryProcessor.java | 47 ++ .../query/processor/CombinedPermissions.java | 25 + .../processor/DeviceSearchQueryProcessor.java | 59 ++ .../processor/DeviceTypeQueryProcessor.java | 47 ++ .../processor/EdgeTypeQueryProcessor.java | 42 ++ .../EdgeTypeSearchQueryProcessor.java | 44 ++ .../EntitiesByGroupNameQueryProcessor.java | 121 ++++ .../EntitiesByGroupQueryProcessor.java | 96 +++ .../EntityGroupListQueryProcessor.java | 84 +++ .../EntityGroupNameQueryProcessor.java | 83 +++ .../processor/EntityListQueryProcessor.java | 94 +++ .../processor/EntityNameQueryProcessor.java | 41 ++ .../query/processor/EntityQueryProcessor.java | 28 + .../EntityQueryProcessorFactory.java | 50 ++ .../processor/EntityTypeQueryProcessor.java | 35 + .../EntityViewSearchQueryProcessor.java | 44 ++ .../EntityViewTypeQueryProcessor.java | 42 ++ .../query/processor/GroupPermissions.java | 29 + .../edqs/query/processor/Permissions.java | 24 + .../processor/RelationQueryPermissions.java | 33 + .../processor/RelationQueryProcessor.java | 84 +++ .../SchedulerEventQueryProcessor.java | 37 ++ .../processor/SingleEntityQueryProcessor.java | 87 +++ .../StateEntityOwnerQueryProcessor.java | 91 +++ .../server/edqs/repo/EdqRepository.java | 44 ++ .../edqs/repo/InMemoryEdqRepository.java | 91 +++ .../server/edqs/repo/KeyDictionary.java | 40 ++ .../server/edqs/repo/TbBytePool.java | 39 ++ .../server/edqs/repo/TbStringPool.java | 37 ++ .../server/edqs/repo/TenantRepo.java | 584 +++++++++++++++++ .../server/edqs/state/EdqsStateService.java | 32 + .../edqs/state/KafkaEdqsStateService.java | 187 ++++++ .../edqs/state/LocalEdqsStateService.java | 81 +++ .../server/edqs/stats/EdqsStatsService.java | 91 +++ .../edqs/util/EdqsPartitionService.java | 40 ++ .../server/edqs/util/EdqsRocksDb.java | 51 ++ .../server/edqs/util/RepositoryUtils.java | 424 ++++++++++++ .../server/edqs/util/TbRocksDb.java | 68 ++ .../server/edqs/util/VersionsStore.java | 48 ++ .../server/common/msg/edqs/EdqsService.java | 47 ++ .../server/common/msg/queue/ServiceType.java | 3 +- .../common/msg/queue/TopicPartitionInfo.java | 22 +- common/pom.xml | 1 + common/proto/src/main/proto/queue.proto | 57 ++ common/queue/pom.xml | 4 + .../AbstractTbQueueConsumerTemplate.java | 12 +- .../common/DefaultTbQueueRequestTemplate.java | 20 +- .../DefaultTbQueueResponseTemplate.java | 14 +- .../consumer/MainQueueConsumerManager.java | 18 +- .../queue/common/consumer}/QueueEvent.java | 2 +- .../consumer}/TbQueueConsumerManagerTask.java | 2 +- .../common/consumer}/TbQueueConsumerTask.java | 12 +- .../DefaultTbServiceInfoProvider.java | 10 + .../queue/discovery/HashPartitionService.java | 76 ++- .../queue/discovery/PartitionService.java | 6 +- .../queue/discovery/ZkDiscoveryService.java | 3 + .../discovery/event/PartitionChangeEvent.java | 19 +- .../server/queue/edqs/EdqsComponent.java | 29 + .../server/queue/edqs/EdqsConfig.java | 55 ++ .../server/queue/edqs/EdqsQueue.java | 36 ++ .../server/queue/edqs/EdqsQueueFactory.java | 35 + .../queue/edqs/InMemoryEdqsComponent.java | 26 + .../queue/edqs/InMemoryEdqsQueueFactory.java | 77 +++ .../server/queue/edqs/KafkaEdqsComponent.java | 28 + .../queue/edqs/KafkaEdqsQueueFactory.java | 122 ++++ .../queue/environment/DistributedLock.java | 24 + .../environment/DistributedLockService.java | 22 + .../DummyDistributedLockService.java | 53 ++ .../environment/ZkDistributedLockService.java | 62 ++ .../server/queue/kafka/KafkaTbQueueMsg.java | 8 +- .../server/queue/kafka/TbKafkaAdmin.java | 8 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 89 ++- .../queue/kafka/TbKafkaProducerTemplate.java | 22 +- .../queue/kafka/TbKafkaTopicConfigs.java | 15 + .../provider/EdqsClientQueueFactory.java | 31 + .../InMemoryMonolithQueueFactory.java | 49 +- .../provider/KafkaMonolithQueueFactory.java | 50 +- .../provider/KafkaTbCoreQueueFactory.java | 48 ++ .../KafkaTbRuleEngineQueueFactory.java | 19 + .../queue/provider/TbCoreQueueFactory.java | 2 +- .../provider/TbQueueProducerProvider.java | 2 +- .../provider/TbRuleEngineQueueFactory.java | 2 +- .../server/queue/util/PropertyUtils.java | 3 +- .../DefaultTbQueueRequestTemplateTest.java | 8 +- .../common/stats/DummyMessagesStats.java | 54 ++ .../server/common/stats/StatsType.java | 3 +- .../thingsboard/common/util/JacksonUtil.java | 4 + .../DefaultClusterVersionControlService.java | 2 +- .../java/org/thingsboard/server/dao/Dao.java | 12 +- .../org/thingsboard/server/dao/DaoUtil.java | 5 + .../server/dao/TenantEntityDao.java | 18 +- .../server/dao/asset/AssetDao.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 7 +- .../server/dao/asset/BaseAssetService.java | 2 +- .../server/dao/attributes/AttributesDao.java | 5 + .../dao/attributes/BaseAttributesService.java | 45 +- .../attributes/CachedAttributesService.java | 20 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../dao/dashboard/DashboardServiceImpl.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../dao/device/DeviceProfileServiceImpl.java | 7 +- .../dao/dictionary/KeyDictionaryDao.java | 5 + .../thingsboard/server/dao/edge/EdgeDao.java | 2 +- .../server/dao/entity/BaseEntityService.java | 73 +++ .../server/dao/entity/EntityDaoRegistry.java | 31 +- .../dao/entityview/EntityViewServiceImpl.java | 2 +- .../dao/eventsourcing/SaveEntityEvent.java | 1 + .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AlarmTypeCompositeKey.java | 33 + .../server/dao/model/sql/AlarmTypeEntity.java | 65 ++ .../model/sqlts/latest/TsKvLatestEntity.java | 9 + .../notification/NotificationTargetDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 + .../dao/queue/BaseQueueStatsService.java | 8 +- .../server/dao/relation/RelationDao.java | 4 + .../server/dao/rule/BaseRuleChainService.java | 12 +- .../server/dao/rule/RuleChainDao.java | 2 +- .../dao/sql/alarm/AlarmCommentRepository.java | 6 +- .../server/dao/sql/alarm/AlarmRepository.java | 2 + .../dao/sql/alarm/AlarmTypeRepository.java | 30 + .../dao/sql/alarm/EntityAlarmRepository.java | 4 + .../dao/sql/alarm/JpaAlarmCommentDao.java | 15 +- .../server/dao/sql/alarm/JpaAlarmDao.java | 14 +- .../server/dao/sql/alarm/JpaAlarmTypeDao.java | 46 ++ .../dao/sql/alarm/JpaEntityAlarmDao.java | 46 ++ .../dao/sql/asset/AssetProfileRepository.java | 5 + .../server/dao/sql/asset/AssetRepository.java | 6 + .../server/dao/sql/asset/JpaAssetDao.java | 17 + .../dao/sql/asset/JpaAssetProfileDao.java | 20 +- .../dao/sql/attributes/JpaAttributeDao.java | 10 + .../dao/sql/customer/CustomerRepository.java | 6 + .../dao/sql/customer/JpaCustomerDao.java | 17 + .../sql/dashboard/DashboardRepository.java | 5 + .../dao/sql/dashboard/JpaDashboardDao.java | 17 + .../device/DeviceCredentialsRepository.java | 6 + .../sql/device/DeviceProfileRepository.java | 6 + .../dao/sql/device/DeviceRepository.java | 6 + .../sql/device/JpaDeviceCredentialsDao.java | 16 +- .../server/dao/sql/device/JpaDeviceDao.java | 17 + .../dao/sql/device/JpaDeviceProfileDao.java | 20 +- .../server/dao/sql/edge/EdgeRepository.java | 5 + .../server/dao/sql/edge/JpaEdgeDao.java | 17 + .../sql/entityview/EntityViewRepository.java | 5 + .../dao/sql/entityview/JpaEntityViewDao.java | 22 +- .../notification/JpaNotificationRuleDao.java | 14 +- .../JpaNotificationTargetDao.java | 11 + .../JpaNotificationTemplateDao.java | 14 +- .../server/dao/sql/ota/JpaOtaPackageDao.java | 19 +- .../dao/sql/ota/JpaOtaPackageInfoDao.java | 1 + .../dao/sql/ota/OtaPackageRepository.java | 6 + .../query/DefaultAlarmQueryRepository.java | 12 +- .../query/DefaultEntityQueryRepository.java | 35 +- .../sql/query/DefaultQueryLogComponent.java | 2 +- .../dao/sql/query/DummyEdqsService.java | 64 ++ .../dao/sql/query/EntityKeyMapping.java | 30 +- .../dao/sql/query/QueryLogComponent.java | 2 +- ...QueryContext.java => SqlQueryContext.java} | 9 +- .../server/dao/sql/queue/JpaQueueDao.java | 14 +- .../dao/sql/queue/JpaQueueStatsDao.java | 6 + .../dao/sql/queue/QueueStatsRepository.java | 5 + .../dao/sql/relation/JpaRelationDao.java | 7 + .../dao/sql/relation/RelationRepository.java | 3 + .../dao/sql/resource/JpaTbResourceDao.java | 9 +- .../server/dao/sql/rpc/JpaRpcDao.java | 14 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 17 + .../server/dao/sql/rule/JpaRuleNodeDao.java | 14 +- .../dao/sql/rule/RuleChainRepository.java | 4 + .../dao/sql/rule/RuleNodeRepository.java | 3 + .../sql/settings/AdminSettingsRepository.java | 4 + .../dao/sql/settings/JpaAdminSettingsDao.java | 17 +- .../server/dao/sql/tenant/JpaTenantDao.java | 12 + .../dao/sql/tenant/JpaTenantProfileDao.java | 6 + .../sql/tenant/TenantProfileRepository.java | 5 + .../dao/sql/tenant/TenantRepository.java | 5 + .../usagerecord/ApiUsageStateRepository.java | 10 + .../sql/usagerecord/JpaApiUsageStateDao.java | 19 + .../dao/sql/user/JpaUserAuthSettingsDao.java | 25 +- .../dao/sql/user/JpaUserCredentialsDao.java | 16 +- .../server/dao/sql/user/JpaUserDao.java | 17 + .../dao/sql/user/JpaUserSettingsDao.java | 17 +- .../sql/user/UserAuthSettingsRepository.java | 5 + .../sql/user/UserCredentialsRepository.java | 6 + .../server/dao/sql/user/UserRepository.java | 6 + .../dao/sql/user/UserSettingsRepository.java | 5 + .../dao/sql/widget/JpaWidgetTypeDao.java | 20 +- .../dao/sql/widget/JpaWidgetsBundleDao.java | 22 +- .../dao/sql/widget/WidgetTypeRepository.java | 5 + .../sql/widget/WidgetsBundleRepository.java | 5 + .../widget/WidgetsBundleWidgetRepository.java | 7 + .../dao/sqlts/AbstractSqlTimeseriesDao.java | 2 +- .../CachedRedisSqlTimeseriesLatestDao.java | 9 + .../dao/sqlts/SqlTimeseriesLatestDao.java | 8 + .../sqlts/dictionary/JpaKeyDictionaryDao.java | 8 + .../dictionary/KeyDictionaryRepository.java | 5 + .../sqlts/latest/TsKvLatestRepository.java | 5 + .../server/dao/tenant/TenantDao.java | 5 +- .../dao/timeseries/BaseTimeseriesService.java | 33 +- .../CassandraBaseTimeseriesLatestDao.java | 9 + .../dao/timeseries/TimeseriesLatestDao.java | 6 + .../dao/usagerecord/ApiUsageStateDao.java | 4 +- .../usagerecord/ApiUsageStateServiceImpl.java | 8 + .../thingsboard/server/dao/user/UserDao.java | 2 +- .../dao/service/AbstractServiceTest.java | 3 +- .../dao/service/EntityDaoRegistryTest.java | 31 + .../query/DefaultQueryLogComponentTest.java | 6 +- edqs/pom.xml | 260 ++++++++ .../edqs/DummyQueueRoutingInfoService.java | 33 + .../edqs/DummyTenantRoutingInfoService.java | 30 + .../edqs/ThingsboardEdqsApplication.java | 119 ++++ edqs/src/main/resources/edqs.yml | 364 +++++++++++ edqs/src/main/resources/logback.xml | 38 ++ .../server/edqs/repo/AbstractEDQTest.java | 296 +++++++++ .../edqs/repo/ApiUsageStateFilterTest.java | 106 +++ .../edqs/repo/AssetSearchQueryFilterTest.java | 223 +++++++ .../server/edqs/repo/AssetTypeFilterTest.java | 187 ++++++ .../repo/DeviceSearchQueryFilterTest.java | 241 +++++++ .../edqs/repo/DeviceTypeFilterTest.java | 142 ++++ .../edqs/repo/EdgeSearchQueryFilterTest.java | 167 +++++ .../server/edqs/repo/EdgeTypeFilterTest.java | 177 +++++ .../repo/EntitiesByGroupIdFilterTest.java | 161 +++++ .../repo/EntitiesByGroupNameFilterTest.java | 140 ++++ .../edqs/repo/EntityGroupListFilterTest.java | 189 ++++++ .../edqs/repo/EntityGroupNameFilterTest.java | 186 ++++++ .../edqs/repo/EntityListFilterTest.java | 202 ++++++ .../edqs/repo/EntityNameFilterTest.java | 132 ++++ .../edqs/repo/EntityTypeFilterTest.java | 147 +++++ .../repo/EntityViewSearchQueryFilterTest.java | 221 +++++++ .../edqs/repo/EntityViewTypeFilterTest.java | 177 +++++ .../edqs/repo/RelationsQueryFilterTest.java | 233 +++++++ .../server/edqs/repo/RepositoryUtilsTest.java | 434 +++++++++++++ .../edqs/repo/SchedulerEventFilterTest.java | 129 ++++ .../edqs/repo/SingleEntityFilterTest.java | 176 +++++ .../edqs/repo/StateEntityOwnerFilterTest.java | 81 +++ edqs/src/test/resources/edq-test.properties | 2 + pom.xml | 18 + 358 files changed, 18294 insertions(+), 675 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java rename {dao/src/test/java/org/thingsboard/server/dao/service => application/src/test/java/org/thingsboard/server/service/entitiy}/EntityServiceTest.java (87%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java rename dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java => common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java (77%) create mode 100644 common/edqs/pom.xml create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java rename {application/src/main/java/org/thingsboard/server/service/queue => common/queue/src/main/java/org/thingsboard/server/queue/common}/consumer/MainQueueConsumerManager.java (95%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/QueueEvent.java (92%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/TbQueueConsumerManagerTask.java (96%) rename {application/src/main/java/org/thingsboard/server/service/queue/ruleengine => common/queue/src/main/java/org/thingsboard/server/queue/common/consumer}/TbQueueConsumerTask.java (89%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java create mode 100644 common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java rename dao/src/main/java/org/thingsboard/server/dao/sql/query/{QueryContext.java => SqlQueryContext.java} (94%) create mode 100644 edqs/pom.xml create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java create mode 100644 edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java create mode 100644 edqs/src/main/resources/edqs.yml create mode 100644 edqs/src/main/resources/logback.xml create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java create mode 100644 edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java create mode 100644 edqs/src/test/resources/edq-test.properties diff --git a/application/pom.xml b/application/pom.xml index d97f6d92e3..9bdfc7c754 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -124,6 +124,10 @@ org.thingsboard.common edge-api + + org.thingsboard.common + edqs + org.thingsboard dao diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 505244539b..f28d485b30 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -27,6 +28,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -38,6 +40,7 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; @@ -55,6 +58,8 @@ public class EntityQueryController extends BaseController { @Autowired private EntityQueryService entityQueryService; + @Autowired + private EdqsService edqsService; private static final int MAX_PAGE_SIZE = 100; @@ -133,4 +138,10 @@ public class EntityQueryController extends BaseController { return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @PostMapping("/edqs/system/request") + public void processSystemEdqsRequest(@RequestBody ToCoreEdqsRequest request) { + edqsService.processSystemRequest(request); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 9317c51a64..c7e95dbbc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -82,6 +82,11 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + try { if (!isValidSaveEntityEventForEdgeProcessing(event)) { return; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java new file mode 100644 index 0000000000..7582d036e2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -0,0 +1,340 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.protobuf.ByteString; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.EdqsSyncRequest; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsCoreServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.environment.DistributedLock; +import org.thingsboard.server.queue.environment.DistributedLockService; +import org.thingsboard.server.queue.provider.EdqsClientQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +@Service +@RequiredArgsConstructor +@Slf4j +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +public class DefaultEdqsService implements EdqsService { + + private final EdqsClientQueueFactory queueFactory; + private final EdqsConverter edqsConverter; + private final EdqsSyncService edqsSyncService; + private final DistributedLockService distributedLockService; + private final AttributesService attributesService; + private final EdqsPartitionService edqsPartitionService; + @Autowired @Lazy + private TbClusterService clusterService; + @Autowired @Lazy + private HashPartitionService hashPartitionService; + + @Value("${queue.edqs.api_enabled:false}") + private Boolean apiEnabled; + + private EdqsProducer eventsProducer; + private TbQueueRequestTemplate, TbProtoQueueMsg> requestTemplate; + private ExecutorService executor; + private DistributedLock syncLock; + + @PostConstruct + private void init() { + executor = ThingsBoardExecutors.newWorkStealingPool(12, getClass()); + eventsProducer = EdqsProducer.builder() + .queue(EdqsQueue.EVENTS) + .partitionService(edqsPartitionService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.EVENTS)) + .build(); + if (apiEnabled) { + apiEnabled = null; + } + + requestTemplate = queueFactory.createEdqsRequestTemplate(); + requestTemplate.init(); + syncLock = distributedLockService.getLock("edqs_sync"); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void onStartUp() { + executor.submit(() -> { + try { + EdqsSyncState syncState = getSyncState(); + if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { + if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { + processSystemRequest(ToCoreEdqsRequest.builder() + .syncRequest(new EdqsSyncRequest()) + .build()); + } + } else { // only if topic/RocksDB is not empty and sync is finished + if (apiEnabled == null) { + log.info("EDQS is already synced, enabling API"); + apiEnabled = true; + } else { + log.info("EDQS is already synced"); + } + } + } catch (Throwable e) { + log.error("Failed to start EDQS service", e); + } + }); + } + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) { + log.info("Processing system request {}", request); + if (request.getSyncRequest() != null) { + saveSyncState(EdqsSyncStatus.REQUESTED); + } + broadcast(request.toInternalMsg()); + } + + @Override + public void processSystemMsg(ToCoreEdqsMsg msg) { + executor.submit(() -> { + log.info("Processing system msg {}", msg); + try { + if (msg.getApiEnabled() != null) { + apiEnabled = msg.getApiEnabled(); + } + + if (msg.getSyncRequest() != null) { + syncLock.lock(); + try { + EdqsSyncState syncState = getSyncState(); + if (syncState != null && syncState.getStatus() == EdqsSyncStatus.FINISHED) { + log.info("EDQS sync is already finished"); + return; + } + + saveSyncState(EdqsSyncStatus.STARTED); + edqsSyncService.sync(); + + saveSyncState(EdqsSyncStatus.FINISHED); + if (apiEnabled == null) { + broadcast(ToCoreEdqsMsg.builder() + .apiEnabled(Boolean.TRUE) + .build()); + } + } catch (Exception e) { + log.error("Failed to complete sync", e); + saveSyncState(EdqsSyncStatus.FAILED); + } finally { + syncLock.unlock(); + } + } + } catch (Throwable e) { + log.error("Failed to process msg {}", msg, e); + } + }); + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring update event, type {} not supported", tenantId, entityId, entityType); + return; + } + onUpdate(tenantId, objectType, edqsConverter.toEntity(entityType, entity)); + } + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.UPDATED, object); + } + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + ObjectType objectType = ObjectType.fromEntityType(entityType); + if (!isEdqsType(tenantId, objectType)) { + log.trace("[{}][{}] Ignoring deletion event, type {} not supported", tenantId, entityId, entityType); + return; + } + onDelete(tenantId, objectType, new Entity(entityType, entityId.getId(), Long.MAX_VALUE)); + } + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) { + processEvent(tenantId, objectType, EdqsEventType.DELETED, object); + } + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + var requestMsg = newEdqsMsg(tenantId) + .setRequestMsg(EdqsRequestMsg.newBuilder() + .setValue(JacksonUtil.toString(request)) + .build()); + if (customerId != null && !customerId.isNullUid()) { + requestMsg.setCustomerIdMSB(customerId.getId().getMostSignificantBits()); + requestMsg.setCustomerIdLSB(customerId.getId().getLeastSignificantBits()); + } + + Integer partition = edqsPartitionService.resolvePartition(tenantId); + ListenableFuture> resultFuture = requestTemplate.send(new TbProtoQueueMsg<>(UUID.randomUUID(), requestMsg.build()), partition); + return Futures.transform(resultFuture, msg -> { + TransportProtos.EdqsResponseMsg responseMsg = msg.getValue().getResponseMsg(); + return JacksonUtil.fromString(responseMsg.getValue(), EdqsResponse.class); + }, MoreExecutors.directExecutor()); + } + + @Override + public boolean isApiEnabled() { + return Boolean.TRUE.equals(apiEnabled); + } + + protected void processEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType, EdqsObject object) { + executor.submit(() -> { + try { + String key = object.key(); + Long version = object.version(); + EdqsEventMsg.Builder eventMsg = EdqsEventMsg.newBuilder() + .setKey(key) + .setObjectType(objectType.name()) + .setData(ByteString.copyFrom(edqsConverter.serialize(objectType, object))) + .setEventType(eventType.name()); + if (version != null) { + eventMsg.setVersion(version); + } + eventsProducer.send(tenantId, objectType, key, newEdqsMsg(tenantId) + .setEventMsg(eventMsg) + .build()); + } catch (Throwable e) { + log.error("[{}] Failed to push {} event for {} {}", tenantId, eventType, objectType, object, e); + } + }); + } + + private boolean isEdqsType(TenantId tenantId, ObjectType objectType) { + if (objectType == null) { + return false; + } + if (!tenantId.isSysTenantId()) { + return ObjectType.edqsTypes.contains(objectType); + } else { + return ObjectType.edqsSystemTypes.contains(objectType); + } + } + + private void broadcast(ToCoreEdqsMsg msg) { + clusterService.broadcastToCore(ToCoreNotificationMsg.newBuilder() + .setToEdqsCoreServiceMsg(ToEdqsCoreServiceMsg.newBuilder() + .setValue(ByteString.copyFrom(JacksonUtil.writeValueAsBytes(msg)))) + .build()); + } + + private static ToEdqsMsg.Builder newEdqsMsg(TenantId tenantId) { + return ToEdqsMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTs(System.currentTimeMillis()); + } + + @PreDestroy + private void preDestroy() { + executor.shutdown(); + eventsProducer.stop(); + requestTemplate.stop(); + } + + @SneakyThrows + private EdqsSyncState getSyncState() { + EdqsSyncState state = attributesService.find(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, "edqsSyncState").get(30, TimeUnit.SECONDS) + .flatMap(KvEntry::getJsonValue) + .map(value -> JacksonUtil.fromString(value, EdqsSyncState.class)) + .orElse(null); + log.info("getSyncState = {}", state); + return state; + } + + @SneakyThrows + private void saveSyncState(EdqsSyncStatus status) { + EdqsSyncState state = new EdqsSyncState(status); + log.info("saveSyncState {}", state); + attributesService.save(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry( + new JsonDataEntry("edqsSyncState", JacksonUtil.toString(state)), + System.currentTimeMillis())).get(30, TimeUnit.SECONDS); + } + + @Data + @AllArgsConstructor + @NoArgsConstructor + private static class EdqsSyncState { + private EdqsSyncStatus status; + } + + private enum EdqsSyncStatus { + REQUESTED, + STARTED, + FINISHED, + FAILED + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java new file mode 100644 index 0000000000..69a3f6108d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java @@ -0,0 +1,539 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.converter.Converter; +import org.thingsboard.server.common.data.converter.ConverterType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.ConverterId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.IntegrationId; +import org.thingsboard.server.common.data.id.RoleId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.SchedulerEventId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.id.WidgetTypeId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.integration.Integration; +import org.thingsboard.server.common.data.integration.IntegrationType; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.role.Role; +import org.thingsboard.server.common.data.role.RoleType; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.scheduler.SchedulerEvent; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.edqs.processor.EdqsConverter; + +import java.io.FileReader; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Consumer; + +import static org.thingsboard.common.util.JacksonUtil.toJsonNode; + + +@RequiredArgsConstructor +@Slf4j +//@Service +public class EdqsDataLoader { + + private final EdqsService edqsService; + private final EdqsConverter edqsConverter; + + public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); + + private final String folder = "/home/viacheslav/Downloads/schwarz"; + + private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher")); + +// @AfterStartUp(order = 100) + public void load() throws Exception { + loadCustomers(); + loadDeviceProfile(); + loadDevices(); + loadAssets(); + loadEdges(); + loadEntityViews(); + loadTenants(); + loadUsers(); + loadDashboards(); + loadRuleChains(); + loadWidgetType(); + loadWidgetBundle(); + loadConverters(); + loadIntegrations(); + loadSchedulerEvents(); + loadRoles(); + loadApiUsageStates(); + loadAssetProfile(); + loadEntityGroups(); + loadRelations(); + + loadAttributes(); + loadTs(); + } + + private void loadCustomers() throws Exception { + load("customer.csv", (values) -> { + Customer customer = new Customer(); + customer.setTitle(values.get("title")); + customer.setId(new CustomerId(UUID.fromString(values.get("id")))); + customer.setCreatedTime(Long.parseLong(values.get("created_time"))); + customer.setTenantId(tenantId(values.get("tenant_id"))); + var parentCustomerId = values.get("parent_customer_id"); + if (StringUtils.isNotEmpty(parentCustomerId)) { + customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId))); + } + edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer); + }); + } + + private void loadDevices() throws Exception { + load("device.csv", (values) -> { + Device device = new Device(); + device.setType(values.get("type")); + device.setName(values.get("name")); + device.setLabel(values.get("label")); + device.setId(new DeviceId(uuid(values.get("id")))); + device.setCreatedTime(parseLong(values.get("created_time"))); + device.setCustomerId(customerId(values.get("customer_id"))); + device.setTenantId(tenantId(values.get("tenant_id"))); + device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id")))); + device.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(device.getTenantId(), device.getId(), device); + }); + } + + private void loadAssets() throws Exception { + load("asset.csv", (values) -> { + Asset asset = new Asset(); + asset.setType(values.get("type")); + asset.setName(values.get("name")); + asset.setLabel(values.get("label")); + asset.setId(new AssetId(uuid(values.get("id")))); + asset.setCreatedTime(parseLong(values.get("created_time"))); + asset.setCustomerId(customerId(values.get("customer_id"))); + asset.setTenantId(tenantId(values.get("tenant_id"))); + asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id")))); + asset.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset); + }); + } + + private void loadEdges() throws Exception { + load("edge.csv", (values) -> { + Edge edge = new Edge(); + edge.setId(new EdgeId(uuid(values.get("id")))); + edge.setCreatedTime(parseLong(values.get("created_time"))); + edge.setType(values.get("type")); + edge.setName(values.get("name")); + edge.setLabel(values.get("label")); + edge.setCustomerId(customerId(values.get("customer_id"))); + edge.setTenantId(tenantId(values.get("tenant_id"))); + edge.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge); + }); + } + + private void loadEntityViews() throws Exception { + load("entity_view.csv", (values) -> { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(uuid(values.get("id")))); + entityView.setCreatedTime(parseLong(values.get("created_time"))); + entityView.setType(values.get("type")); + entityView.setName(values.get("name")); + entityView.setCustomerId(customerId(values.get("customer_id"))); + entityView.setTenantId(tenantId(values.get("tenant_id"))); + entityView.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView); + }); + } + + private void loadTenants() throws Exception { + load("tenant.csv", (values) -> { + Tenant tenant = new Tenant(); + tenant.setId(new TenantId(uuid(values.get("id")))); + tenant.setCreatedTime(parseLong(values.get("created_time"))); + tenant.setEmail(values.get("email")); + tenant.setTitle(values.get("title")); + tenant.setCountry(values.get("country")); + tenant.setState(values.get("state")); + tenant.setCity(values.get("city")); + tenant.setAddress(values.get("address")); + tenant.setAddress2(values.get("address2")); + tenant.setZip(values.get("zip")); + tenant.setPhone(values.get("phone")); + tenant.setRegion(values.get("region")); + tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id")))); + tenant.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + edqsService.onUpdate(MAIN, tenant.getId(), tenant); + }); + } + + private void loadUsers() throws Exception { + load("user.csv", (values) -> { + User user = new User(); + user.setId(new UserId(uuid(values.get("id")))); + user.setCreatedTime(parseLong(values.get("created_time"))); + user.setTenantId(tenantId(values.get("tenant_id"))); + user.setFirstName(values.get("first_name")); + user.setLastName(values.get("last_name")); + user.setEmail(values.get("email")); + user.setPhone(values.get("phone")); + user.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(user.getTenantId(), user.getId(), user); + }); + } + + private void loadDashboards() throws Exception { + load("dashboard.csv", (values) -> { + Dashboard dashboard = new Dashboard(); + dashboard.setId(new DashboardId(uuid(values.get("id")))); + dashboard.setCreatedTime(parseLong(values.get("created_time"))); + dashboard.setTenantId(tenantId(values.get("tenant_id"))); + dashboard.setTitle(values.get("title")); + + edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard); + }); + } + + private void loadEntityGroups() throws Exception { + load("entity_group.csv", (values) -> { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(uuid(values.get("id")))); + entityGroup.setCreatedTime(parseLong(values.get("created_time"))); + entityGroup.setName(values.get("name")); + entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id"))); + entityGroup.setType(EntityType.valueOf(values.get("type"))); + edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup); + }); + } + + private void loadRelations() throws Exception { + load("relation.csv", (values) -> { + EntityRelation entityRelation = new EntityRelation(); + entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id"))); + entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id"))); + entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group"))); + entityRelation.setType(values.get("relation_type")); + edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation); + }); + } + + private void loadRuleChains() throws Exception { + load("rule_chain.csv", (values) -> { + RuleChain ruleChain = new RuleChain(); + ruleChain.setId(new RuleChainId(uuid(values.get("id")))); + ruleChain.setCreatedTime(parseLong(values.get("created_time"))); + ruleChain.setName(values.get("name")); + ruleChain.setTenantId(tenantId(values.get("tenant_id"))); + ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain); + }); + } + + private void loadWidgetType() throws Exception { + load("widget_type.csv", (values) -> { + WidgetType widgetType = new WidgetType(); + widgetType.setId(new WidgetTypeId(uuid(values.get("id")))); + widgetType.setCreatedTime(parseLong(values.get("created_time"))); + widgetType.setName(values.get("name")); + widgetType.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType); + }); + } + + private void loadWidgetBundle() throws Exception { + load("widgets_bundle.csv", (values) -> { + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id")))); + widgetsBundle.setCreatedTime(parseLong(values.get("created_time"))); + widgetsBundle.setTitle(values.get("title")); + widgetsBundle.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle); + }); + } + + private void loadConverters() throws Exception { + load("converter.csv", (values) -> { + Converter converter = new Converter(); + converter.setId(new ConverterId(uuid(values.get("id")))); + converter.setCreatedTime(parseLong(values.get("created_time"))); + converter.setName(values.get("name")); + converter.setType(ConverterType.valueOf(values.get("type"))); + converter.setTenantId(tenantId(values.get("tenant_id"))); + converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); + converter.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter); + }); + } + + private void loadIntegrations() throws Exception { + load("integration.csv", (values) -> { + Integration integration = new Integration(); + integration.setId(new IntegrationId(uuid(values.get("id")))); + integration.setCreatedTime(parseLong(values.get("created_time"))); + integration.setName(values.get("name")); + integration.setType(IntegrationType.valueOf(values.get("type"))); + integration.setTenantId(tenantId(values.get("tenant_id"))); + integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); + integration.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration); + }); + } + + private void loadSchedulerEvents() throws Exception { + load("scheduler_event.csv", (values) -> { + SchedulerEvent schedulerEvent = new SchedulerEvent(); + schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id")))); + schedulerEvent.setCreatedTime(parseLong(values.get("created_time"))); + schedulerEvent.setName(values.get("name")); + schedulerEvent.setType(values.get("type")); + schedulerEvent.setTenantId(tenantId(values.get("tenant_id"))); + schedulerEvent.setConfiguration(toJsonNode(values.get("configuration"))); + schedulerEvent.setSchedule(toJsonNode(values.get("schedule"))); + schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id"))); + schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent); + }); + } + + private void loadRoles() throws Exception { + load("role.csv", (values) -> { + Role role = new Role(); + role.setId(new RoleId(uuid(values.get("id")))); + role.setCreatedTime(parseLong(values.get("created_time"))); + role.setName(values.get("name")); + role.setType(RoleType.valueOf(values.get("type"))); + role.setTenantId(tenantId(values.get("tenant_id"))); + role.setAdditionalInfo(toJsonNode(values.get("additional_info"))); + + edqsService.onUpdate(role.getTenantId(), role.getId(), role); + }); + } + + private void loadApiUsageStates() throws Exception { + load("api_usage_state.csv", (values) -> { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id")))); + apiUsageState.setCreatedTime(parseLong(values.get("created_time"))); + apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id"))); + apiUsageState.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState); + }); + } + + private void loadDeviceProfile() throws Exception { + load("device_profile.csv", (values) -> { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setId(new DeviceProfileId(uuid(values.get("id")))); + deviceProfile.setCreatedTime(parseLong(values.get("created_time"))); + deviceProfile.setName(values.get("name")); + deviceProfile.setType(DeviceProfileType.valueOf(values.get("type"))); + deviceProfile.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile); + }); + } + + private void loadAssetProfile() throws Exception { + load("asset_profile.csv", (values) -> { + AssetProfile assetProfile = new AssetProfile(); + assetProfile.setId(new AssetProfileId(uuid(values.get("id")))); + assetProfile.setCreatedTime(parseLong(values.get("created_time"))); + assetProfile.setName(values.get("name")); + assetProfile.setTenantId(tenantId(values.get("tenant_id"))); + + edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile); + }); + } + + private void loadAttributes() throws Exception { + load("attribute.csv", (values) -> { + EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id")); + long ts = parseLong(values.get("last_update_ts")); + AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type")); + String key = values.get("attribute_key"); + KvEntry kvEntry; + if (StringUtils.isNotEmpty(values.get("bool_v"))) { + kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); + } else if (StringUtils.isNotEmpty(values.get("str_v"))) { + kvEntry = new StringDataEntry(key, values.get("str_v")); + } else if (StringUtils.isNotEmpty(values.get("long_v"))) { + kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); + } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { + kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); + } else if (StringUtils.isNotEmpty(values.get("json_v"))) { + kvEntry = new JsonDataEntry(key, values.get("json_v")); + } else { + kvEntry = new StringDataEntry(key, ""); + } + AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry); + AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0); + edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv); + }); + } + + private void loadTs() throws Exception { + load("ts_kv.csv", (values) -> { + var entityTypeStr = values.get("find_entity_type"); + if (StringUtils.isEmpty(entityTypeStr)) { + return; + } + EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id")); + long ts = parseLong(values.get("ts")); + String key = values.get("key"); + KvEntry kvEntry; + if (StringUtils.isNotEmpty(values.get("bool_v"))) { + kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); + } else if (StringUtils.isNotEmpty(values.get("str_v"))) { + kvEntry = new StringDataEntry(key, values.get("str_v")); + } else if (StringUtils.isNotEmpty(values.get("long_v"))) { + kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); + } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { + kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); + } else if (StringUtils.isNotEmpty(values.get("json_v"))) { + kvEntry = new JsonDataEntry(key, values.get("json_v")); + } else { + kvEntry = new StringDataEntry(key, ""); + } + BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry); + edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L)); + }); + } + + private void load(String file, Consumer> function) throws Exception { + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> { + try { + long ts = System.currentTimeMillis(); + CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|'); + CsvMapper mapper = new CsvMapper(); + MappingIterator> it = mapper + .readerFor(Map.class) + .with(schema) + .readValues(new FileReader(folder + "/" + file)); + + int success = 0; + int failure = 0; + while (it.hasNextValue()) { + Map row = it.nextValue(); + try { + function.accept(row); + success++; + if (success % 1000 == 0) { + log.info("Loaded [{}] from [{}]", success, file); + } + } catch (Exception e) { + log.error("Failed to parse str: [{}]", row, e); + failure++; + } + } + log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure); + } catch (Throwable t) { + log.error("Failed to load data from [{}]", file, t); + } + }); + } + + private static TenantId tenantId(String id) { + return TenantId.fromUUID(UUID.fromString(id)); + } + + private static CustomerId customerId(String id) { + var c = new CustomerId(UUID.fromString(id)); + return c.isNullUid() ? null : c; + } + + private static EntityId entityId(String type, String id) { + return EntityIdFactory.getByTypeAndId(type, id); + } + + private static UUID uuid(String id) { + return UUID.fromString(id); + } + + private static long parseLong(String time) { + return Long.parseLong(time); + } + + private static boolean parseBoolean(String value) { + return Boolean.parseBoolean(value); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java new file mode 100644 index 0000000000..c4ce0c7a7e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +public class EdqsListener { + + private final EdqsService edqsService; + + @TransactionalEventListener(fallbackExecution = true) + public void onUpdate(SaveEntityEvent event) { + if (event.getEntityId() == null || event.getEntity() == null) { + return; + } + edqsService.onUpdate(event.getTenantId(), event.getEntityId(), event.getEntity()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void onDelete(DeleteEntityEvent event) { + if (event.getEntityId() == null) { + return; + } + edqsService.onDelete(event.getTenantId(), event.getEntityId()); + } + + @TransactionalEventListener(fallbackExecution = true) + public void handleEvent(RelationActionEvent relationEvent) { + if (relationEvent.getActionType() == ActionType.RELATION_ADD_OR_UPDATE) { + edqsService.onUpdate(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } else if (relationEvent.getActionType() == ActionType.RELATION_DELETED) { + edqsService.onDelete(relationEvent.getTenantId(), ObjectType.RELATION, relationEvent.getRelation()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java new file mode 100644 index 0000000000..8e12da56e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -0,0 +1,275 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.group.EntityGroupDao; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; + +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.thingsboard.server.common.data.ObjectType.API_USAGE_STATE; +import static org.thingsboard.server.common.data.ObjectType.ASSET; +import static org.thingsboard.server.common.data.ObjectType.ASSET_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.BLOB_ENTITY; +import static org.thingsboard.server.common.data.ObjectType.CONVERTER; +import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; +import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; +import static org.thingsboard.server.common.data.ObjectType.DEVICE; +import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.EDGE; +import static org.thingsboard.server.common.data.ObjectType.ENTITY_GROUP; +import static org.thingsboard.server.common.data.ObjectType.ENTITY_VIEW; +import static org.thingsboard.server.common.data.ObjectType.INTEGRATION; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.QUEUE_STATS; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.ROLE; +import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; +import static org.thingsboard.server.common.data.ObjectType.SCHEDULER_EVENT; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.USER; +import static org.thingsboard.server.common.data.ObjectType.WIDGETS_BUNDLE; +import static org.thingsboard.server.common.data.ObjectType.WIDGET_TYPE; + +@Slf4j +public abstract class EdqsSyncService { + + @Autowired + private EntityDaoRegistry entityDaoRegistry; + @Autowired + private TenantDao tenantDao; + @Autowired + private AttributesDao attributesDao; + @Autowired + private KeyDictionaryDao keyDictionaryDao; + @Autowired + private RelationDao relationDao; + @Autowired + private EntityGroupDao entityGroupDao; + @Autowired + private TimeseriesLatestDao timeseriesLatestDao; + @Autowired + @Lazy + private DefaultEdqsService edqsService; + + private final ConcurrentHashMap entityInfoMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap keys = new ConcurrentHashMap<>(); + + private final Map counters = new ConcurrentHashMap<>(); + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, + BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + ); + + public abstract boolean isSyncNeeded(); + + public void sync() { + log.info("Synchronizing data to EDQS"); + long startTs = System.currentTimeMillis(); + counters.clear(); + + syncTenants(); + syncTenantEntities(); + syncEntityGroups(); + syncRelations(); + loadKeyDictionary(); + syncAttributes(); + syncLatestTimeseries(); + + counters.clear(); + log.info("Finishing synchronizing data to EDQS in {} ms", (System.currentTimeMillis() - startTs)); + } + + private void process(TenantId tenantId, ObjectType type, EdqsObject object) { + AtomicInteger counter = counters.computeIfAbsent(type, t -> new AtomicInteger()); + if (counter.incrementAndGet() % 10000 == 0) { + log.info("Processed {} {} objects", counter.get(), type); + } + edqsService.processEvent(tenantId, type, EdqsEventType.UPDATED, object); + } + + private void syncTenants() { + log.info("Synchronizing tenants to EDQS"); + long ts = System.currentTimeMillis(); + var tenants = new PageDataIterable<>(tenantDao::findAllFields, 10000); + for (EntityFields entityFields : tenants) { + TenantId tenantId = TenantId.fromUUID(entityFields.getId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(EntityType.TENANT, tenantId)); + process(tenantId, TENANT, new Entity(EntityType.TENANT, entityFields)); + } + process(TenantId.SYS_TENANT_ID, TENANT, new Entity(EntityType.TENANT, new TenantFields(TenantId.SYS_TENANT_ID.getId(), Long.MAX_VALUE))); + log.info("Finished synchronizing tenants to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncTenantEntities() { + for (ObjectType type : edqsTenantTypes) { + log.info("Synchronizing tenant {} entities to EDQS", type); + long ts = System.currentTimeMillis(); + EntityType entityType = type.toEntityType(); + Dao dao = entityDaoRegistry.getDao(entityType); + var entities = new PageDataIterable<>(dao::findAllFields, 10000); + for (EntityFields entityFields : entities) { + TenantId tenantId = TenantId.fromUUID(entityFields.getTenantId()); + entityInfoMap.put(entityFields.getId(), new EntityIdInfo(entityType, tenantId)); + process(tenantId, type, new Entity(type.toEntityType(), entityFields)); + } + log.info("Finished synchronizing tenant {} entities to EDQS in {} ms", type, (System.currentTimeMillis() - ts)); + } + } + + private void syncEntityGroups() { + log.info("Synchronizing entity groups to EDQS"); + long ts = System.currentTimeMillis(); + var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 10000); + for (EntityFields groupFields : entityGroups) { + EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); + if (entityIdInfo != null) { + entityInfoMap.put(groupFields.getId(), new EntityIdInfo(EntityType.ENTITY_GROUP, entityIdInfo.tenantId())); + process(entityIdInfo.tenantId(), ENTITY_GROUP, new Entity(EntityType.ENTITY_GROUP, groupFields)); + } else { + log.info("Entity group owner not found: " + groupFields.getOwnerId()); + } + } + log.info("Finished synchronizing entity groups to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncRelations() { + log.info("Synchronizing relations to EDQS"); + long ts = System.currentTimeMillis(); + var relations = new PageDataIterable<>(relationDao::findAll, 10000); + for (EntityRelation relation : relations) { + if (relation.getTypeGroup() == RelationTypeGroup.COMMON || relation.getTypeGroup() == RelationTypeGroup.FROM_ENTITY_GROUP) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFrom().getId()); + if (entityIdInfo != null) { + process(entityIdInfo.tenantId(), RELATION, relation); + } else { + log.info("Relation from entity not found: " + relation.getFrom()); + } + } + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void loadKeyDictionary() { + log.info("Loading key dictionary"); + long ts = System.currentTimeMillis(); + var keyDictionaryEntries = new PageDataIterable<>(keyDictionaryDao::findAll, 10000); + for (KeyDictionaryEntry keyDictionaryEntry : keyDictionaryEntries) { + keys.put(keyDictionaryEntry.getKeyId(), keyDictionaryEntry.getKey()); + } + log.info("Finished loading key dictionary in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncAttributes() { + log.info("Synchronizing attributes to EDQS"); + long ts = System.currentTimeMillis(); + var attributes = new PageDataIterable<>(attributesDao::findAll, 10000); + for (AttributeKvEntity attribute : attributes) { + attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); + UUID entityId = attribute.getId().getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); + if (entityIdInfo == null) { + log.debug("Skipping attribute with entity UUID {} as it is not found in entityInfoMap", entityId); + continue; + } + AttributeKv attributeKv = new AttributeKv( + EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityId), + AttributeScope.valueOf(attribute.getId().getAttributeType()), + attribute.toData(), + attribute.getVersion()); + process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void syncLatestTimeseries() { + log.info("Synchronizing latest timeseries to EDQS"); + long ts = System.currentTimeMillis(); + var tsKvLatestEntities = new PageDataIterable<>(pageLink -> timeseriesLatestDao.findAllLatest(pageLink), 10000); + for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { + try { + String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); + if (strKey == null) { + log.debug("Skipping latest timeseries with key {} as it is not found in key dictionary", tsKvLatestEntity.getKey()); + continue; + } + tsKvLatestEntity.setStrKey(strKey); + UUID entityUuid = tsKvLatestEntity.getEntityId(); + EntityIdInfo entityIdInfo = entityInfoMap.get(entityUuid); + if (entityIdInfo != null) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityIdInfo.entityType(), entityUuid); + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvLatestEntity.toData(), tsKvLatestEntity.getVersion()); + process(entityIdInfo.tenantId(), LATEST_TS_KV, latestTsKv); + } + } catch (Exception e) { + log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); + } + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private String getStrKeyOrFetchFromDb(int key) { + String strKey = keys.get(key); + if (strKey != null) { + return strKey; + } else { + strKey = keyDictionaryDao.getKey(key); + keys.putIfAbsent(key, strKey); + } + return strKey; + } + + public record EntityIdInfo(EntityType entityType, TenantId tenantId) {} + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java new file mode 100644 index 0000000000..d51d87ed01 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; + +import java.util.Collections; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +public class KafkaEdqsSyncService extends EdqsSyncService { + + private final TbKafkaSettings kafkaSettings; + private TbKafkaAdmin kafkaAdmin; + + @PostConstruct + private void init() { + kafkaAdmin = new TbKafkaAdmin(kafkaSettings, Collections.emptyMap()); + } + + @Override + public boolean isSyncNeeded() { + return kafkaAdmin.isTopicEmpty(EdqsQueue.STATE.getTopic()); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java new file mode 100644 index 0000000000..924bf94830 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +public class LocalEdqsSyncService extends EdqsSyncService { + + private final EdqsRocksDb db; + + @Override + public boolean isSyncNeeded() { + return db.isNew(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 80368c085f..3cc999df29 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -72,6 +72,11 @@ public class EntityStateSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(SaveEntityEvent event) { + if (Boolean.FALSE.equals(event.getBroadcastEvent())) { + log.trace("Ignoring event {}", event); + return; + } + TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); if (entityId == null) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 3aaa36b068..1b4bfe7c71 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.Data; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -35,6 +34,7 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.alarm.AlarmInfo; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -78,6 +79,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -89,7 +91,6 @@ import org.thingsboard.server.service.notification.NotificationSchedulerService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.resource.TbImageService; @@ -147,9 +148,10 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; private QueueConsumerManager> usageStatsConsumer; private QueueConsumerManager> firmwareStatesConsumer; @@ -175,7 +177,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, CoreQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE)) - .config(CoreQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCoreMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -251,7 +255,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, CoreQueueConfig config) throws Exception { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig config) throws Exception { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -396,6 +400,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService, EdgeQueueConfig> mainConsumer; + private MainQueueConsumerManager, QueueConfig> mainConsumer; public DefaultTbEdgeConsumerService(TbCoreQueueFactory tbCoreQueueFactory, ActorSystemContext actorContext, StatsFactory statsFactory, EdgeContextComponent edgeCtx) { @@ -102,9 +102,9 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService, EdgeQueueConfig>builder() + this.mainConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.TB_CORE).withQueueName(DataConstants.EDGE_QUEUE_NAME)) - .config(EdgeQueueConfig.of(consumerPerPartition, pollInterval)) + .config(QueueConfig.of(consumerPerPartition, pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) @@ -130,7 +130,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService> msgs, TbQueueConsumer> consumer, EdgeQueueConfig edgeQueueConfig) throws InterruptedException { + private void processMsgs(List> msgs, TbQueueConsumer> consumer, QueueConfig edgeQueueConfig) throws InterruptedException { List> orderedMsgList = msgs.stream().map(msg -> new IdMsgPair<>(UUID.randomUUID(), msg)).toList(); ConcurrentMap> pendingMap = orderedMsgList.stream().collect( Collectors.toConcurrentMap(IdMsgPair::getUuid, IdMsgPair::getMsg)); @@ -323,10 +323,4 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService { + event.getNewPartitions().forEach((queueKey, partitions) -> { if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { var consumer = getConsumer(queueKey).orElseGet(() -> { Queue config = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..3636eb05af 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -33,11 +33,14 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.QueueEvent; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerManagerTask; +import org.thingsboard.server.queue.common.consumer.TbQueueConsumerTask; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; -import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index c58cb4fd5a..d3d0421297 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -251,7 +251,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService= deviceState.getLastActivityTime()) { deviceState.setLastInactivityAlarmTime(0L); - save(deviceId, INACTIVITY_ALARM_TIME, 0L); + save(state.getTenantId(), deviceId, INACTIVITY_ALARM_TIME, 0L); } } } @@ -583,7 +583,7 @@ public class DefaultDeviceStateService extends AbstractPartitionBasedService>> customExporters; + private Map relatedEntitiesExporters; + + private static final Set RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV); + + @PostConstruct + private void init() { + relatedEntitiesExporters = Map.of( + RELATION, this::exportRelations, + EVENT, this::exportEvents, // todo: query by tenant + ATTRIBUTE_KV, this::exportAttributes, + LATEST_TS_KV, this::exportLatestTelemetry + ); + customExporters = Map.of( + AUDIT_LOG, this::exportAuditLogs + ); + } + + public void exportTenant(TenantId tenantId, ExportConfig config, BiConsumer processor) { + log.info("[{}] Exporting tenant", tenantId); + Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (tenant == null) { + throw new IllegalArgumentException("Tenant with id " + tenantId + " not found"); + } + + Set objectTypes = config.getIncludedObjectTypes(); + if (objectTypes.contains(TENANT)) { + exportEntity(tenantId, TENANT, tenant, config, processor); + } + + for (ObjectType type : objectTypes) { + if (RELATED.contains(type) || type == TENANT) { + continue; + } + log.debug("[{}] Exporting {} entities", tenantId, type); + if (!customExporters.containsKey(type)) { + TenantEntityDao dao = entityDaoRegistry.getTenantEntityDao(type); + var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100); + for (Object entity : entities) { + exportEntity(tenantId, type, entity, config, processor); + } + } else { + customExporters.get(type).accept(tenantId, processor); + } + } + } + + private void exportEntity(TenantId tenantId, ObjectType type, Object entity, ExportConfig config, BiConsumer processor) { + processor.accept(type, entity); + if (entity instanceof HasId hasId && hasId.getId() instanceof EntityId entityId) { + relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> { + if (config.getIncludedObjectTypes().contains(relatedEntityType)) { + exporter.export(tenantId, entityId, processor); + } + }); + } + } + + private Map getPartitions(String table) { + List partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList(); + if (partitionsStartTime.isEmpty()) { + return Collections.emptyMap(); + } + + Map partitions = new HashMap<>(); + for (int i = 0; i < partitionsStartTime.size(); i++) { + Long startTime = partitionsStartTime.get(i); + Long endTime; + if (partitionsStartTime.size() - 1 == i) { + endTime = System.currentTimeMillis(); + } else { + endTime = partitionsStartTime.get(i + 1) - 1; + } + partitions.put(startTime, endTime); + } + return partitions; + } + + private void exportAuditLogs(TenantId tenantId, BiConsumer processor) { + Map partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME); + partitions.forEach((startTime, endTime) -> { + PageDataIterable auditLogs = new PageDataIterable<>(pageLink -> { + return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (AuditLog auditLog : auditLogs) { + processor.accept(AUDIT_LOG, auditLog); + } + }); + } + + private void exportAttributes(TenantId tenantId, EntityId entityId, BiConsumer processor) { + for (AttributeScope attributeScope : AttributeScope.values()) { + List attributes = attributesDao.findAll(tenantId, entityId, attributeScope); + for (AttributeKvEntry entry : attributes) { + AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry, entry.getVersion()); + processor.accept(ATTRIBUTE_KV, attributeKv); + } + } + } + + private void exportRelations(TenantId tenantId, EntityId entityId, BiConsumer processor) { + List relations = relationDao.findAllByFrom(tenantId, entityId); + for (EntityRelation relation : relations) { + processor.accept(RELATION, relation); + } + } + + @SneakyThrows + private void exportLatestTelemetry(TenantId tenantId, EntityId entityId, BiConsumer processor) { + List latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS); + for (TsKvEntry tsKvEntry : latestTelemetry) { + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry, tsKvEntry.getVersion()); + processor.accept(LATEST_TS_KV, latestTsKv); + } + } + + private void exportEvents(TenantId tenantId, EntityId entityId, BiConsumer processor) { + for (EventType eventType : EventType.values()) { + Map partitions = getPartitions(eventType.getTable()); + partitions.forEach((startTime, endTime) -> { + PageDataIterable events = new PageDataIterable<>(pageLink -> { + return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (Event event : events) { + processor.accept(EVENT, event); + } + }); + } + } + + private interface Exporter { + + void export(TenantId tenantId, EntityId entityId, BiConsumer processor); + + } + + @Data + public static class ExportConfig { + + private Set includedObjectTypes; + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index b38fbf4fec..c0360ede8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -93,7 +93,8 @@ public class TbCoreTransportApiService { @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { log.info("Received application ready event. Starting polling for events."); - transportApiTemplate.init(transportApiService); + transportApiTemplate.subscribe(); + transportApiTemplate.launch(transportApiService); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java index 34fd4a426f..2bd39b8c09 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/DefaultWebSocketService.java @@ -229,6 +229,7 @@ public class DefaultWebSocketService implements WebSocketService { } catch (TbRateLimitsException e) { log.debug("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } catch (Exception e) { + sendError(sessionRef, cmd.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR, e.getMessage()); log.error("{} Failed to handle WS cmd: {}", sessionRef, cmd, e); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f29cc55197..9b182ce3a7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1615,6 +1615,12 @@ queue: edge: "${TB_QUEUE_KAFKA_EDGE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1688,6 +1694,23 @@ queue: enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release + api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release + mode: "${TB_EDQS_MODE:local}" # local or remote + local: + rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}" + partitions: "${TB_EDQS_PARTITIONS:12}" + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + stats: + # Enable/disable statistics for EDQS service + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" vc: # Default topic name diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java new file mode 100644 index 0000000000..daa1c92c1c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.type=kafka", // uncomment to use Kafka + "queue.kafka.bootstrap.servers=10.7.1.254:9092", + "queue.edqs.sync_enabled=true", + "queue.edqs.api_enabled=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { + + @Autowired + private EdqsService edqsService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void before() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); + } + + @Override + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> findByQuery(query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) { + return await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == expectedResult); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 5d04b16dfa..ad554f11ba 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -18,12 +18,14 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.awaitility.Awaitility; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -49,6 +51,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; @@ -73,6 +76,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -130,36 +134,25 @@ public class EntityQueryControllerTest extends AbstractControllerTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); - countQuery = new EntityCountQuery(entityListFilter); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); EntityTypeFilter filter2 = new EntityTypeFilter(); filter2.setEntityType(EntityType.DEVICE); - - EntityCountQuery countQuery2 = new EntityCountQuery(filter2); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class); - Assert.assertEquals(97, count2.longValue()); + countQuery = new EntityCountQuery(filter2); + countByQueryAndCheck(countQuery, 97); } @Test @@ -169,14 +162,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); allDeviceFilter.setEntityType(EntityType.DEVICE); EntityCountQuery query = new EntityCountQuery(allDeviceFilter); - Long initialCount = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); + Long initialCount = countByQuery(query); loginTenantAdmin(); List devices = new ArrayList<>(); + String devicePrefix = "Device" + RandomStringUtils.random(5); for (int i = 0; i < 97; i++) { Device device = new Device(); - device.setName("Device" + i); + device.setName(devicePrefix + i); device.setType("default"); device.setLabel("testLabel" + (int) (Math.random() * 1000)); devices.add(doPost("/api/device", device, Device.class)); @@ -189,31 +183,23 @@ public class EntityQueryControllerTest extends AbstractControllerTest { loginSysAdmin(); EntityCountQuery countQuery = new EntityCountQuery(filter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, initialCount + 97); filter.setDeviceType("unknown"); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(0, count.longValue()); + countByQueryAndCheck(countQuery, 0); filter.setDeviceType("default"); - filter.setDeviceNameFilter("Device1"); - - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(11, count.longValue()); + filter.setDeviceNameFilter(devicePrefix + "1"); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); + countByQueryAndCheck(countQuery, 97); - count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); - - Long count2 = doPostWithResponse("/api/entitiesQuery/count", query, Long.class); - Assert.assertEquals(initialCount + 97, count2.longValue()); + countByQueryAndCheck(query, initialCount + 97); } @Test @@ -371,11 +357,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -383,8 +365,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -406,8 +387,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -423,9 +403,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null); - PageData data2 = - doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference>() { - }); + PageData data2 = findByQuery(query2); Assert.assertEquals(97, data2.getTotalElements()); Assert.assertEquals(10, data2.getTotalPages()); @@ -473,20 +451,15 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), false))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(10, data.getTotalElements()); + findByQueryAndCheck(query, 10); filter.setFilters(List.of(new RelationEntityTypeFilter("NOT_CONTAINS", List.of(EntityType.DEVICE), true))); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); - Assert.assertEquals(87, data.getTotalElements()); + findByQueryAndCheck(query, 87); } private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { @@ -531,14 +504,12 @@ public class EntityQueryControllerTest extends AbstractControllerTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -551,6 +522,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(45)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); @@ -559,13 +531,11 @@ public class EntityQueryControllerTest extends AbstractControllerTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -604,6 +574,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, "alarmActiveTime")); + highTemperatureFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); DynamicValue dynamicValue = @@ -627,16 +598,16 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - Awaitility.await() + await() .alias("data by query") .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == numOfDevices; }); - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(numOfDevices, loadedEntities.size()); @@ -694,11 +665,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(entityTypeFilter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -712,9 +679,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { }); EntityCountQuery countQuery = new EntityCountQuery(entityTypeFilter); - - Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); - Assert.assertEquals(97, count.longValue()); + countByQueryAndCheck(countQuery, 97); } @Test @@ -742,28 +707,28 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeToLongFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 30); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); // all devices with ownerName = TEST TENANT - EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - checkEntitiesCount(query, numOfDevices); + EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(query, numOfDevices); // all devices with ownerName = TEST TENANT - EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); - checkEntitiesCount(activeAlarmTimeToLongQuery, 0); + EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); + countByQueryAndCheck(activeAlarmTimeToLongQuery, 0); // all devices with wrong ownerName EntityCountQuery wrongTenantNameQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, wrongOwnerNameFilter)); - checkEntitiesCount(wrongTenantNameQuery, 0); + countByQueryAndCheck(wrongTenantNameQuery, 0); // all devices with owner type = TENANT EntityCountQuery tenantEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerTypeFilter)); - checkEntitiesCount(tenantEntitiesQuery, numOfDevices); + countByQueryAndCheck(tenantEntitiesQuery, numOfDevices); // all devices with owner type = CUSTOMER EntityCountQuery customerEntitiesQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, customerOwnerTypeFilter)); - checkEntitiesCount(customerEntitiesQuery, 0); + countByQueryAndCheck(customerEntitiesQuery, 0); } @Test @@ -790,7 +755,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { KeyFilter activeAlarmTimeFilter = getServerAttributeNumericGreaterThanKeyFilter("alarmActiveTime", 5); KeyFilter tenantOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", TEST_TENANT_NAME); KeyFilter wrongOwnerNameFilter = getEntityFieldStringEqualToKeyFilter("ownerName", "wrongName"); - KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); + KeyFilter tenantOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "TENANT"); KeyFilter customerOwnerTypeFilter = getEntityFieldStringEqualToKeyFilter("ownerType", "CUSTOMER"); EntityDataSortOrder sortOrder = new EntityDataSortOrder( @@ -874,18 +839,18 @@ public class EntityQueryControllerTest extends AbstractControllerTest { } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { - Awaitility.await() + await() .alias("data by query") .atMost(30, TimeUnit.SECONDS) .until(() -> { - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); return loadedEntities.size() == expectedNumOfDevices; }); if (expectedNumOfDevices == 0) { return; } - var data = doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() {}); + var data = findByQuery(query); var loadedEntities = new ArrayList<>(data.getData()); Assert.assertEquals(expectedNumOfDevices, loadedEntities.size()); @@ -898,25 +863,36 @@ public class EntityQueryControllerTest extends AbstractControllerTest { String alarmActiveTime = entity.getLatest().get(EntityKeyType.ATTRIBUTE).getOrDefault("alarmActiveTime", new TsValue(0, "-1")).getValue(); Assert.assertEquals("Device" + i, name); - Assert.assertEquals( expectedOwnerName, ownerName); - Assert.assertEquals( expectedOwnerType, ownerType); + Assert.assertEquals(expectedOwnerName, ownerName); + Assert.assertEquals(expectedOwnerType, ownerType); Assert.assertEquals("1" + i, alarmActiveTime); } } - private void checkEntitiesCount(EntityCountQuery query, int expectedNumOfDevices) { - Awaitility.await() - .alias("count by query") - .atMost(30, TimeUnit.SECONDS) - .until(() -> { - var count = doPost("/api/entitiesQuery/count", query, Integer.class); - return count == expectedNumOfDevices; - }); - } + protected PageData findByQuery(EntityDataQuery query) throws Exception { + return doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference<>() {}); + } + + protected PageData findByQueryAndCheck(EntityDataQuery query, int expectedResultSize) throws Exception { + PageData result = findByQuery(query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected Long countByQuery(EntityCountQuery countQuery) throws Exception { + return doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + } + + protected Long countByQueryAndCheck(EntityCountQuery query, long expectedResult) throws Exception { + Long result = countByQuery(query); + assertThat(result).isEqualTo(expectedResult); + return result; + } private KeyFilter getEntityFieldStringEqualToKeyFilter(String keyName, String value) { KeyFilter tenantOwnerNameFilter = new KeyFilter(); tenantOwnerNameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, keyName)); + tenantOwnerNameFilter.setValueType(EntityKeyValueType.STRING); StringFilterPredicate ownerNamePredicate = new StringFilterPredicate(); ownerNamePredicate.setValue(FilterPredicateValue.fromString(value)); ownerNamePredicate.setOperation(StringFilterPredicate.StringOperation.EQUAL); @@ -927,6 +903,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { private KeyFilter getServerAttributeNumericGreaterThanKeyFilter(String attribute, int value) { KeyFilter numericFilter = new KeyFilter(); numericFilter.setKey(new EntityKey(EntityKeyType.SERVER_ATTRIBUTE, attribute)); + numericFilter.setValueType(EntityKeyValueType.NUMERIC); NumericFilterPredicate predicate = new NumericFilterPredicate(); predicate.setValue(FilterPredicateValue.fromDouble(value)); predicate.setOperation(NumericFilterPredicate.NumericOperation.GREATER); diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 52cbaa4add..0a4009e8b2 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -153,7 +153,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()); + ServiceInfo serviceInfo = partitionService.resolveByPartitionIdx(services, queueKey, partition, Collections.emptyMap()).get(0); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -308,7 +308,7 @@ public class HashPartitionServiceTest { partitionService_common.recalculatePartitions(commonRuleEngine, List.of(dedicatedRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, TenantId.SYS_TENANT_ID); - return event.getPartitionsMap().get(queueKey).size() == systemQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == systemQueue.getPartitions(); }); Mockito.reset(applicationEventPublisher); @@ -336,14 +336,14 @@ public class HashPartitionServiceTest { // expecting event about no partitions for isolated queue key verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); partitionService_dedicated.updateQueues(List.of(queueUpdateMsg)); partitionService_dedicated.recalculatePartitions(dedicatedRuleEngine, List.of(commonRuleEngine)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).size() == isolatedQueue.getPartitions(); + return event.getNewPartitions().get(queueKey).size() == isolatedQueue.getPartitions(); }); @@ -361,7 +361,7 @@ public class HashPartitionServiceTest { partitionService_dedicated.removeQueues(List.of(queueDeleteMsg)); verifyPartitionChangeEvent(event -> { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.MAIN_QUEUE_NAME, tenantId); - return event.getPartitionsMap().get(queueKey).isEmpty(); + return event.getNewPartitions().get(queueKey).isEmpty(); }); } @@ -381,12 +381,12 @@ public class HashPartitionServiceTest { Stream.concat(Stream.of(TenantId.SYS_TENANT_ID), Stream.generate(UUID::randomUUID).map(TenantId::new).limit(10)).forEach(tenantId -> { List queues = Stream.generate(() -> RandomStringUtils.randomAlphabetic(10)) .map(queueName -> new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId)) - .limit(100).collect(Collectors.toList()); + .limit(100).toList(); for (int partition = 0; partition < 10; partition++) { - ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()); + ServiceInfo expectedAssignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId), partition, Collections.emptyMap()).get(0); for (QueueKey queueKey : queues) { - ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()); + ServiceInfo assignedRuleEngine = partitionService.resolveByPartitionIdx(ruleEngines, queueKey, partition, Collections.emptyMap()).get(0); assertThat(assignedRuleEngine).as(queueKey + "[" + partition + "] should be assigned to " + expectedAssignedRuleEngine.getServiceId()) .isEqualTo(expectedAssignedRuleEngine); } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java new file mode 100644 index 0000000000..eea7e0eb7f --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy; + +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.edqs.util.EdqsRocksDb; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +@DaoSqlTest +@TestPropertySource(properties = { + "queue.edqs.sync_enabled=true", + "queue.edqs.api_enabled=true", + "queue.edqs.mode=local" +}) +public class EdqsEntityServiceTest extends EntityServiceTest { + + @Autowired + private EdqsService edqsService; + + @MockBean + private EdqsRocksDb edqsRocksDb; + + @Before + public void beforeEach() { + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); + } + + @Override + protected PageData findByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityDataQuery query, long expectedResultSize) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, permissions, query), + result -> result.getTotalElements() == expectedResultSize); + } + + @Override + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), mergedUserPermissionsPE, countQuery, expectedResult); + } + + @Override + protected long countByQueryAndCheck(CustomerId customerId, MergedUserPermissions permissions, EntityCountQuery query, int expectedResult) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> countByQuery(customerId, permissions, query), + result -> result == expectedResult); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java similarity index 87% rename from dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java rename to application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index a26f345fb7..dc29cec3cc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -13,22 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.service; +package org.thingsboard.server.service.entitiy; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; @@ -37,6 +40,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -46,6 +50,7 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.objects.TelemetryEntityView; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; @@ -63,6 +68,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -75,17 +81,22 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardDao; +import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewDao; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; @@ -111,7 +122,7 @@ import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIEL @Slf4j @DaoSqlTest -public class EntityServiceTest extends AbstractServiceTest { +public class EntityServiceTest extends AbstractControllerTest { static final int ENTITY_COUNT = 5; public static final String TEST_CUSTOMER_NAME = "Test"; @@ -119,6 +130,12 @@ public class EntityServiceTest extends AbstractServiceTest { @Autowired AssetService assetService; @Autowired + AssetProfileService assetProfileService; + @Autowired + DashboardService dashboardService; + @Autowired + EntityViewService entityViewService; + @Autowired UserService userService; @Autowired AttributesService attributesService; @@ -157,7 +174,7 @@ public class EntityServiceTest extends AbstractServiceTest { } @Test - public void testCountEntitiesByQuery() throws InterruptedException { + public void testCountEntitiesByQuery() { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -173,30 +190,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDeviceNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setDeviceTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setDeviceTypes(List.of("default")); filter.setDeviceNameFilter("Device1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.DEVICE); entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); deviceService.deleteDevicesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @@ -211,19 +222,15 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(31, count); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) + countByQueryAndCheck(countQuery, 31); //due to the loop relations in hierarchy, the TenantId included in total count (1*Tenant + 5*Asset + 5*5*Devices = 31) filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Contains", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter.setRootEntity(devices.get(0).getId()); filter.setDirection(EntitySearchDirection.TO); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("Manages", Collections.singletonList(EntityType.TENANT)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(1, count); + countByQueryAndCheck(countQuery, 1); DeviceSearchQueryFilter filter2 = new DeviceSearchQueryFilter(); filter2.setRootEntity(tenantId); @@ -231,18 +238,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter2.setRelationType("Contains"); countQuery = new EntityCountQuery(filter2); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(25, count); + countByQueryAndCheck(countQuery, 25); filter2.setDeviceTypes(Arrays.asList("default0", "default1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(10, count); + countByQueryAndCheck(countQuery, 10); filter2.setRootEntity(devices.get(0).getId()); filter2.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); AssetSearchQueryFilter filter3 = new AssetSearchQueryFilter(); filter3.setRootEntity(tenantId); @@ -250,18 +253,14 @@ public class EntityServiceTest extends AbstractServiceTest { filter3.setRelationType("Manages"); countQuery = new EntityCountQuery(filter3); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter3.setAssetTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); filter3.setRootEntity(devices.get(0).getId()); filter3.setDirection(EntitySearchDirection.TO); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -278,11 +277,12 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData entityDataByQuery = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData entityDataByQuery = findByQueryAndCheck(query, 5); List data = entityDataByQuery.getData(); Assert.assertEquals(data.size(), 5); data.forEach(entityData -> Assert.assertNotNull(entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("phone"))); + countByQueryAndCheck(query, 5); } private void createTestUserRelations(TenantId tenantId, List users) { @@ -312,30 +312,24 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setEdgeNameFilter(""); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); filter.setEdgeTypes(List.of("unknown")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); filter.setEdgeTypes(List.of("default")); filter.setEdgeNameFilter("Edge1"); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(11, count); + countByQueryAndCheck(countQuery, 11); EntityListFilter entityListFilter = new EntityListFilter(); entityListFilter.setEntityType(EntityType.EDGE); entityListFilter.setEntityList(edges.stream().map(Edge::getId).map(EdgeId::toString).collect(Collectors.toList())); countQuery = new EntityCountQuery(entityListFilter); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(97, count); + countByQueryAndCheck(countQuery, 97); edgeService.deleteEdgesByTenantId(tenantId); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(0, count); + countByQueryAndCheck(countQuery, 0); } @Test @@ -360,13 +354,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setRelationType("Manages"); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(5, count); + countByQueryAndCheck(countQuery, 5); filter.setEdgeTypes(Arrays.asList("type0", "type1")); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(2, count); + countByQueryAndCheck(countQuery, 2); } private Edge createEdge(int i, String type) { @@ -424,11 +415,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 25); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(25, loadedEntities.size()); @@ -437,6 +428,9 @@ public class EntityServiceTest extends AbstractServiceTest { List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); Assert.assertEquals(deviceTemperatures, loadedTemperatures); + //count query + countByQueryAndCheck(query, 25); + pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = new KeyFilter(); highTemperatureFilter.setKey(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); @@ -447,13 +441,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFilters = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -464,6 +457,9 @@ public class EntityServiceTest extends AbstractServiceTest { Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); + //count query + countByQueryAndCheck(query, deviceHighTemperatures.size()); + deviceService.deleteDevicesByTenantId(tenantId); } @@ -482,13 +478,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setDirection(EntitySearchDirection.FROM); EntityCountQuery countQuery = new EntityCountQuery(filter); - - long count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(63, count); + countByQueryAndCheck(countQuery, 63); filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(27, count); + countByQueryAndCheck(countQuery, 27); filter.setMultiRootEntitiesType(EntityType.ASSET); filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); @@ -496,13 +489,10 @@ public class EntityServiceTest extends AbstractServiceTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - - count = entityService.countEntitiesByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), countQuery); - Assert.assertEquals(9, count); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); - } @Test @@ -538,15 +528,6 @@ public class EntityServiceTest extends AbstractServiceTest { onlineStatusFilter.setPredicate(predicate); List keyFilters = Collections.singletonList(onlineStatusFilter); - EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - loadedEntities.addAll(data.getData()); - } - long expectedEntitiesCnt = entityNameByTypeMap.entrySet() .stream() .filter(e -> !e.getKey().equals("building")) @@ -554,6 +535,14 @@ public class EntityServiceTest extends AbstractServiceTest { .map(Map.Entry::getValue) .filter(e -> StringUtils.endsWith(e, "_1")) .count(); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + PageData data = findByQueryAndCheck(query, expectedEntitiesCnt); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } Assert.assertEquals(expectedEntitiesCnt, loadedEntities.size()); Map actualRelations = new HashMap<>(); @@ -603,11 +592,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 25); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(25, loadedEntities.size()); @@ -628,12 +617,12 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -676,11 +665,11 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.ATTRIBUTE, "consumption")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 5); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(5, loadedEntities.size()); @@ -700,12 +689,12 @@ public class EntityServiceTest extends AbstractServiceTest { query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highConsumptions.size(), loadedEntities.size()); @@ -896,9 +885,7 @@ public class EntityServiceTest extends AbstractServiceTest { List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - - Assert.assertEquals(97, data.getTotalElements()); + PageData data = findByQueryAndCheck(query, 97); Assert.assertEquals(10, data.getTotalPages()); Assert.assertTrue(data.hasNext()); Assert.assertEquals(10, data.getData().size()); @@ -906,7 +893,7 @@ public class EntityServiceTest extends AbstractServiceTest { List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(97, loadedEntities.size()); @@ -931,7 +918,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(10, 0, "device1", sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); Assert.assertEquals(11, data.getTotalElements()); Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); @@ -945,11 +932,12 @@ public class EntityServiceTest extends AbstractServiceTest { devices.get(1).setLabel(null); devices.forEach(deviceService::saveDevice); + // FIXME (for Dasha, plz investigate): + // this and other tests below submit an empty value to a KEY FILTER, this is not "search text". + // why are we supposed to ignore it and return all devices? maybe it's a bug? String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -961,9 +949,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = devices.get(2).getLabel(); EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size() - 1, result.getTotalElements()); + findByQueryAndCheck(query, devices.size() - 1); } @Test @@ -976,8 +962,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_EQUAL, searchQuery); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -989,9 +974,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.STARTS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1003,9 +986,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.ENDS_WITH, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1017,9 +998,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1031,9 +1010,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = "label-"; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(2, result.getTotalElements()); + findByQueryAndCheck(query, 2); } @Test @@ -1045,9 +1022,7 @@ public class EntityServiceTest extends AbstractServiceTest { String searchQuery = ""; EntityDataQuery query = createDeviceSearchQuery("label", StringOperation.NOT_CONTAINS, searchQuery); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); } @Test @@ -1071,34 +1046,27 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test public void testFindEntityDataByQuery_filter_entity_name_ends_with() { List devices = new ArrayList<>(); + String suffixes = RandomStringUtils.randomAlphanumeric(5); for (int i = 0; i < 10; i++) { Device device = new Device(); device.setTenantId(tenantId); - device.setName("Device " + i + " test"); + device.setName("Device " + i + suffixes); device.setType("default"); devices.add(device); } @@ -1107,29 +1075,21 @@ public class EntityServiceTest extends AbstractServiceTest { EntityNameFilter deviceTypeFilter = new EntityNameFilter(); deviceTypeFilter.setEntityType(EntityType.DEVICE); - deviceTypeFilter.setEntityNameFilter("%test"); + deviceTypeFilter.setEntityNameFilter("%" + suffixes); EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + findByQueryAndCheck(query, devices.size()); - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); - - deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter("%" + suffixes + "%"); + findByQueryAndCheck(query, devices.size()); - deviceTypeFilter.setEntityNameFilter("test"); + deviceTypeFilter.setEntityNameFilter(suffixes + "%"); + findByQueryAndCheck(query, 0); - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + deviceTypeFilter.setEntityNameFilter(suffixes); + findByQueryAndCheck(query, 0); } @Test @@ -1153,19 +1113,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setEntityNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setEntityNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1189,24 +1143,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%Device"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1239,24 +1185,6 @@ public class EntityServiceTest extends AbstractServiceTest { assertThat(deviceName).isEqualTo(devices.get(0).getName()); } - @Test - public void testFindEntitiesByApiUsageStateFilter() { - apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); - apiUsageStateFilter.setCustomerId(customerId); - - List entityFields = List.of( - new EntityKey(EntityKeyType.ENTITY_FIELD, "name") - ); - - EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); - EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); - String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); - assertThat(name).isEqualTo(TEST_CUSTOMER_NAME); - } - @Test public void testFindEntitiesByRelationEntityTypeFilter() { Customer customer = new Customer(); @@ -1341,24 +1269,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1382,19 +1302,13 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(devices.size(), result.getTotalElements()); + findByQueryAndCheck(query, devices.size()); deviceTypeFilter.setDeviceNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); deviceTypeFilter.setDeviceNameFilter("%test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1418,24 +1332,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%Asset"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1459,24 +1365,16 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("%test%"); - - result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("test"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); } @Test @@ -1488,6 +1386,7 @@ public class EntityServiceTest extends AbstractServiceTest { asset.setTenantId(tenantId); asset.setName("Asset test" + i); asset.setType("default"); + asset.setAssetProfileId(assetProfileService.findDefaultAssetProfile(tenantId).getId()); assets.add(asset); } @@ -1500,19 +1399,107 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); - - PageData result = searchEntities(query); - assertEquals(assets.size(), result.getTotalElements()); + findByQueryAndCheck(query, assets.size()); assetTypeFilter.setAssetNameFilter("test%"); - - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + findByQueryAndCheck(query, 0); assetTypeFilter.setAssetNameFilter("%test"); + findByQueryAndCheck(query, 0); + } + + @Test + public void testFindEntitiesBySingleEntityFilter_customer() { + List customerDevices = new ArrayList<>(); + List tenantDevices = new ArrayList<>(); + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test" + i); + device.setType("default"); + Device saved = deviceService.saveDevice(device); + customerDevices.add(saved); + } - result = searchEntities(query); - assertEquals(0, result.getTotalElements()); + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Tenant test device" + i); + device.setType("default"); + tenantDevices.add(deviceService.saveDevice(device)); + } + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(customerDevices.get(0).getId()); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); + + PageData result = findByQueryAndCheck(query, 1); + String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + + // try to find tenant device by customer user + SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); + tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); + EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); + PageData customerResults2 = entityService.findEntityDataByQuery(tenantId, customerId, customerQuery2); + + assertEquals(0, customerResults2.getTotalElements()); + + // find by tenant user with group permission + PageData results3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + + assertEquals(1, results3.getTotalElements()); + String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); + } + + private List getResultDeviceIds(PageData result) { + return result.getData().stream().map(entityData -> (DeviceId) entityData.getEntityId()).collect(Collectors.toList()); + } + + private Device createDevice(CustomerId customerId) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName("Device test " + RandomStringUtils.randomAlphabetic(5)); + device.setType("default"); + return device; + } + + @Test + public void testFindEntitiesByApiUsageStateFilter() { + ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); + PageData result = findByQueryAndCheck(query, 1); + String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(name).isEqualTo(TEST_TENANT_NAME); + + // find by customer user with generic permissions + apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); + PageData customerResult = entityService.findEntityDataByQuery(tenantId, customerId, query); + + assertEquals(1, customerResult.getTotalElements()); + String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); + + // find by tenant user with customerId filter + apiUsageStateFilter.setCustomerId(customerId); + PageData tenantResult = searchEntities(query); + assertEquals(1, tenantResult.getTotalElements()); + String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } private PageData searchEntities(EntityDataQuery query) { @@ -1597,11 +1584,11 @@ public class EntityServiceTest extends AbstractServiceTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -1618,14 +1605,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -1718,7 +1703,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterTemperature); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, greaterTemperatures.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(greaterTemperatures.size(), loadedEntities.size()); @@ -1732,7 +1717,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersGreaterOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, greaterOrEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(greaterOrEqualTemperatures.size(), loadedEntities.size()); @@ -1746,7 +1731,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, lessTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(lessTemperatures.size(), loadedEntities.size()); @@ -1760,7 +1745,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersLessOrEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, lessOrEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(lessOrEqualTemperatures.size(), loadedEntities.size()); @@ -1774,7 +1759,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, equalTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalTemperatures.size(), loadedEntities.size()); @@ -1788,7 +1773,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualTemperature); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualTemperatures.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualTemperatures.size(), loadedEntities.size()); @@ -1843,12 +1828,12 @@ public class EntityServiceTest extends AbstractServiceTest { List latestValues = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, 67); List loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(67, loadedEntities.size()); @@ -1871,13 +1856,12 @@ public class EntityServiceTest extends AbstractServiceTest { List keyFilters = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); - - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, highTemperatures.size()); loadedEntities = new ArrayList<>(data.getData()); while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); @@ -1994,7 +1978,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, equalStrings.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(equalStrings.size(), loadedEntities.size()); @@ -2007,7 +1991,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notEqualStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notEqualStrings.size(), loadedEntities.size()); @@ -2020,7 +2004,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, startsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(startsWithStrings.size(), loadedEntities.size()); @@ -2033,7 +2017,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, endsWithStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(endsWithStrings.size(), loadedEntities.size()); @@ -2046,7 +2030,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, containsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(containsStrings.size(), loadedEntities.size()); @@ -2059,7 +2043,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, notContainsStrings.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(notContainsStrings.size(), loadedEntities.size()); @@ -2072,7 +2056,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, deviceTypeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2117,7 +2101,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEqualString); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2132,7 +2116,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotEqualString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2145,7 +2129,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersStartsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2158,7 +2142,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersEndsWithString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2171,7 +2155,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2184,7 +2168,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, keyFiltersNotContainsString); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2232,7 +2216,7 @@ public class EntityServiceTest extends AbstractServiceTest { EntityDataPageLink pageLink = new EntityDataPageLink(100, 0, null, sortOrder); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, deviceTypeFilters); - PageData data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + PageData data = findByQueryAndCheck(query, devices.size()); List loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2240,7 +2224,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, sortOrder); query = new EntityDataQuery(filter, pageLink, entityFields, null, createdTimeFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2248,7 +2232,7 @@ public class EntityServiceTest extends AbstractServiceTest { pageLink = new EntityDataPageLink(100, 0, null, null); query = new EntityDataQuery(filter, pageLink, entityFields, null, nameFilters); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQueryAndCheck(query, devices.size()); loadedEntities = getLoadedEntities(data, query); Assert.assertEquals(devices.size(), loadedEntities.size()); @@ -2296,12 +2280,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing EntityDataPageLink originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); EntityDataQuery originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, deviceTypeFilters); - PageData originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + PageData originalData = findByQueryAndCheck(originalQuery, expectedDevicesSize); // query without textSearch - optimization is performing EntityDataPageLink optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); EntityDataQuery optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, deviceTypeFilters); - PageData optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + PageData optimizedData = findByQueryAndCheck(optimizedQuery, expectedDevicesSize); List loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2325,12 +2309,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, attributeFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, attributeFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2354,12 +2338,12 @@ public class EntityServiceTest extends AbstractServiceTest { // query with textSearch - optimization is not performing originalPageLink = new EntityDataPageLink(pageSize, 0, "Device", sortOrder); originalQuery = new EntityDataQuery(filter, originalPageLink, entityFields, null, nameFilters); - originalData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), originalQuery); + originalData = findByQuery(originalQuery); // query without textSearch - optimization is performing optimizedPageLink = new EntityDataPageLink(pageSize, 0, null, sortOrder); optimizedQuery = new EntityDataQuery(filter, optimizedPageLink, entityFields, null, nameFilters); - optimizedData = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), optimizedQuery); + optimizedData = findByQuery(optimizedQuery); loadedEntities = getLoadedEntities(optimizedData, optimizedQuery); Assert.assertEquals(expectedDevicesSize, loadedEntities.size()); loadedEntities = getLoadedEntities(originalData, originalQuery); @@ -2387,10 +2371,9 @@ public class EntityServiceTest extends AbstractServiceTest { private List getLoadedEntities(PageData data, EntityDataQuery query) { List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { query = query.next(); - data = entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); + data = findByQuery(query); loadedEntities.addAll(data.getData()); } return loadedEntities; @@ -2421,13 +2404,13 @@ public class EntityServiceTest extends AbstractServiceTest { private ListenableFuture> saveLongAttribute(EntityId entityId, String key, long value, AttributeScope scope) { KvEntry attrValue = new LongDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture> saveStringAttribute(EntityId entityId, String key, String value, AttributeScope scope) { KvEntry attrValue = new StringDataEntry(key, value); AttributeKvEntry attr = new BaseAttributeKvEntry(attrValue, 42L); - return attributesService.save(SYSTEM_TENANT_ID, entityId, scope, Collections.singletonList(attr)); + return attributesService.save(tenantId, entityId, scope, Collections.singletonList(attr)); } private ListenableFuture saveLongTimeseries(EntityId entityId, String key, Double value) { @@ -2436,7 +2419,7 @@ public class EntityServiceTest extends AbstractServiceTest { tsKv.setDoubleValue(value); KvEntry telemetryValue = new DoubleDataEntry(key, value); BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, telemetryValue); - return timeseriesService.save(SYSTEM_TENANT_ID, entityId, timeseries); + return timeseriesService.save(tenantId, entityId, timeseries); } private void createMultiRootHierarchy(List buildings, List apartments, @@ -2510,4 +2493,78 @@ public class EntityServiceTest extends AbstractServiceTest { } } } + + @Test + public void testFindEntitiesWithEntityViewFilter() { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setCustomerId(customerId); + entityView.setName("test"); + entityView.setType("default"); + entityView.setEntityId(new DeviceId(UUID.randomUUID())); + entityView.setKeys(new TelemetryEntityView(List.of("test"), null)); + entityView.setStartTimeMs(124); + entityView.setEndTimeMs(256); + entityView.setExternalId(new EntityViewId(UUID.randomUUID())); + entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); + entityView = entityViewDao.save(tenantId, entityView); + + EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); + entityViewTypeFilter.setEntityViewNameFilter("test"); + entityViewTypeFilter.setEntityViewTypes(List.of("non-existing", "default")); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); + + PageData relationsResult = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult.getData()).hasSize(1); + assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); + + // find with non existing name + entityViewTypeFilter.setEntityViewNameFilter("non-existing"); + PageData relationsResult2 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult2.getData()).hasSize(0); + + // find with non existing type + entityViewTypeFilter.setEntityViewNameFilter(null); + entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); + + PageData relationsResult3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + assertThat(relationsResult3.getData()).hasSize(0); + } + + private PageData findByQuery(EntityDataQuery query) { + return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); + } + + protected PageData findByQuery(CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query); + } + + private PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); + } + + protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { + PageData result = entityService.findEntityDataByQuery(tenantId, customerId, query); + assertThat(result.getTotalElements()).isEqualTo(expectedResultSize); + return result; + } + + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { + return entityService.countEntitiesByQuery(tenantId, customerId, query); + } + + protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { + return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); + } + + protected long countByQueryAndCheck(CustomerId customerId, EntityCountQuery query, int expectedResult) { + long result = countByQuery(customerId, query); + assertThat(result).isEqualTo(expectedResult); + return result; + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java index 0b6e70b84c..449eca3e47 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManagerTest.java @@ -626,8 +626,7 @@ public class TbRuleEngineQueueConsumerManagerTest { .until(() -> consumer.subscribed && consumer.getPartitions().equals(expectedPartitions) && consumer.pollingStarted); verify(consumer, times(1)).subscribe(any()); verify(consumer).subscribe(eq(expectedPartitions)); - verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions.stream() - .map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList())))); + verify(consumer).doSubscribe(argThat(topics -> topics.containsAll(expectedPartitions))); verify(consumer, atLeastOnce()).poll(eq((long) queue.getPollInterval())); verify(consumer, atLeastOnce()).doPoll(eq((long) queue.getPollInterval())); verify(consumer, never()).unsubscribe(); @@ -743,9 +742,11 @@ public class TbRuleEngineQueueConsumerManagerTest { } @Override - protected void doSubscribe(List topicNames) { - log.debug("doSubscribe({})", topicNames); - this.topics = topicNames; + protected void doSubscribe(Set partitions) { + this.topics = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toList()); + log.debug("doSubscribe({})", topics); subscribed = true; } diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index b58d19e0e5..9880ec964b 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -556,7 +556,7 @@ public class DefaultDeviceStateServiceTest { .thenReturn(new PageData<>(List.of(deviceIdInfo), 0, 1, false)); PartitionChangeEvent event = new PartitionChangeEvent(this, ServiceType.TB_CORE, Map.of( new QueueKey(ServiceType.TB_CORE), Collections.singleton(tpi) - )); + ), Collections.emptyMap()); service.onApplicationEvent(event); Thread.sleep(100); } diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 9951caa876..ba6863c705 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -53,6 +53,9 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 -server.log_controller_error_stack_trace=false +server.log_controller_error_stack_trace=true transport.gateway.dashboard.sync.enabled=false + +queue.edqs.sync_enabled=false +queue.edqs.api_enabled=false diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 13c93da411..a0efcf52c1 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -9,7 +9,7 @@ - + diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java index 103d7fe9cd..1b14ec4a23 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueRequestTemplate.java @@ -26,6 +26,8 @@ public interface TbQueueRequestTemplate send(Request request, long timeoutNs); + ListenableFuture send(Request request, Integer partition); + void stop(); void setMessagesStats(MessagesStats messagesStats); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java index 641b4d8faf..bf77c8501e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueResponseTemplate.java @@ -15,9 +15,17 @@ */ package org.thingsboard.server.queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; + +import java.util.Set; + public interface TbQueueResponseTemplate { - void init(TbQueueHandler handler); + void subscribe(); + + void subscribe(Set partitions); + + void launch(TbQueueHandler handler); void stop(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java new file mode 100644 index 0000000000..c78998cd6a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +public enum ObjectType { + TENANT, + TENANT_PROFILE, + CUSTOMER, + ADMIN_SETTINGS, + QUEUE, + RPC, + RULE_CHAIN, + OTA_PACKAGE, + RESOURCE, + ROLE, + ENTITY_GROUP, + DEVICE_GROUP_OTA_PACKAGE, + GROUP_PERMISSION, + BLOB_ENTITY, + SCHEDULER_EVENT, + EVENT, + RULE_NODE, + CONVERTER, + INTEGRATION, + USER, + USER_CREDENTIALS, + USER_AUTH_SETTINGS, + EDGE, + WIDGETS_BUNDLE, + WIDGET_TYPE, + DASHBOARD, + DEVICE_PROFILE, + DEVICE, + DEVICE_CREDENTIALS, + ASSET_PROFILE, + ASSET, + ENTITY_VIEW, + ALARM, + ENTITY_ALARM, + OAUTH2_CLIENT, + OAUTH2_DOMAIN, + OAUTH2_MOBILE, + USER_SETTINGS, + NOTIFICATION_TARGET, + NOTIFICATION_TEMPLATE, + NOTIFICATION_RULE, + WHITE_LABELING, + CUSTOM_TRANSLATION, + ALARM_COMMENT, + ALARM_TYPE, + API_USAGE_STATE, + QUEUE_STATS, + + AUDIT_LOG, + RELATION, + ATTRIBUTE_KV, + LATEST_TS_KV; + + public static final Set edqsTenantTypes = EnumSet.of( + TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, CONVERTER, INTEGRATION, SCHEDULER_EVENT, ROLE, + BLOB_ENTITY, API_USAGE_STATE, QUEUE_STATS + ); + public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); + public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, + API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + + static { + edqsTypes.addAll(Arrays.asList(TENANT, ENTITY_GROUP, RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + } + + public EntityType toEntityType() { + return EntityType.valueOf(name()); + } + + public static ObjectType fromEntityType(EntityType entityType) { + try { + return ObjectType.valueOf(entityType.name()); + } catch (Exception e) { + return null; + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java new file mode 100644 index 0000000000..0813e72123 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class AlarmType { + + private TenantId tenantId; + private String type; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java new file mode 100644 index 0000000000..f2c35466dc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/AttributeKv.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class AttributeKv implements EdqsObject { + + private EntityId entityId; + private AttributeScope scope; + private String key; + private Long version; + + private Long lastUpdateTs; // optional (on deletion) + private KvEntry value; // optional (on deletion) + + public AttributeKv(EntityId entityId, AttributeScope scope, AttributeKvEntry attributeKvEntry, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = attributeKvEntry.getKey(); + this.version = version; + this.lastUpdateTs = attributeKvEntry.getLastUpdateTs(); + this.value = attributeKvEntry; + } + + public AttributeKv(EntityId entityId, AttributeScope scope, String key, long version) { + this.entityId = entityId; + this.scope = scope; + this.key = key; + this.version = version; + } + + @Override + public String key() { + return "a_" + entityId + "_" + scope + "_" + key; + } + + @Override + public Long version() { + return version; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java new file mode 100644 index 0000000000..b070121baa --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEvent.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@AllArgsConstructor +@Builder +public class EdqsEvent { + + private final TenantId tenantId; + private final ObjectType objectType; + private final EdqsEventType eventType; + private final EdqsObject object; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java new file mode 100644 index 0000000000..2907691f91 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsEventType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +public enum EdqsEventType { + UPDATED, + DELETED +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java new file mode 100644 index 0000000000..9a2836149a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsObject.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +public interface EdqsObject { + + @JsonIgnore + String key(); + + @JsonIgnore + Long version(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java new file mode 100644 index 0000000000..69bb4b8888 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/EdqsSyncRequest.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +@Data +@JsonIgnoreProperties +public class EdqsSyncRequest { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java new file mode 100644 index 0000000000..decfbbd116 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/Entity.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityIdFields; + +import java.util.UUID; + +@Data +@NoArgsConstructor +public class Entity implements EdqsObject { + + private EntityType type; + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + private EntityFields fields; + + public Entity(EntityType type) { + this.type = type; + } + + public Entity(EntityType type, EntityFields fields) { + this.type = type; + this.fields = fields; + } + + public Entity(EntityType entityType, UUID id, long version) { + this.type = entityType; + this.fields = new EntityIdFields(id, version); + } + + @Override + public String key() { + return "e_" + fields.getId().toString(); + } + + @Override + public Long version() { + return fields.getVersion(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java new file mode 100644 index 0000000000..8aaabad1a7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/LatestTsKv.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class LatestTsKv implements EdqsObject { + + private EntityId entityId; + private String key; + private Long version; + + private Long ts; // optional (on deletion) + private KvEntry value; // optional (on deletion) + + public LatestTsKv(EntityId entityId, TsKvEntry tsKvEntry, Long version) { + this.entityId = entityId; + this.key = tsKvEntry.getKey(); + this.ts = tsKvEntry.getTs(); + this.version = version != null ? version : 0L; + this.value = tsKvEntry; + } + + public LatestTsKv(EntityId entityId, String key, Long version) { + this.entityId = entityId; + this.key = key; + this.version = version != null ? version : 0L; + } + + public String key() { + return "l_" + entityId + "_" + key; + } + + @Override + public Long version() { + return version; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java new file mode 100644 index 0000000000..802a9c23cf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsMsg.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class ToCoreEdqsMsg { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java new file mode 100644 index 0000000000..e5382a43c1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/ToCoreEdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class ToCoreEdqsRequest { + + private EdqsSyncRequest syncRequest; + private Boolean apiEnabled; + + @JsonIgnore + public ToCoreEdqsMsg toInternalMsg() { + return new ToCoreEdqsMsg(syncRequest, apiEnabled); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java new file mode 100644 index 0000000000..f520e60bdc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AbstractEntityFields.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; + +@Data +@SuperBuilder +public class AbstractEntityFields implements EntityFields { + + private UUID id; + private long createdTime; + private UUID tenantId; + private UUID customerId; + private String name; + private Long version; + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + this.id = id; + this.createdTime = createdTime; + this.tenantId = tenantId; + this.customerId = (customerId != null && customerId != CustomerId.NULL_UUID) ? customerId : null; + this.name = name; + this.version = version; + } + + public AbstractEntityFields() { + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + this(id, createdTime, tenantId, null, name, version); + } + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version) { + this(id, createdTime, tenantId, customerId, null, version); + + } + + public AbstractEntityFields(UUID id, long createdTime, String name, Long version) { + this(id, createdTime, null, name, version); + } + + + public AbstractEntityFields(UUID id, long createdTime, UUID tenantId) { + this(id, createdTime, tenantId, null, null, null); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java new file mode 100644 index 0000000000..f81604cfbc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ApiUsageStateFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@SuperBuilder +public class ApiUsageStateFields extends AbstractEntityFields { + + private EntityId entityId; + private ApiUsageStateValue transportState; + private ApiUsageStateValue dbStorageState; + private ApiUsageStateValue reExecState; + private ApiUsageStateValue jsExecState; + private ApiUsageStateValue tbelExecState; + private ApiUsageStateValue emailExecState; + private ApiUsageStateValue smsExecState; + private ApiUsageStateValue alarmExecState; + + public ApiUsageStateFields(UUID id, long createdTime, UUID tenantId, UUID entityId, String entityType, ApiUsageStateValue transportState, ApiUsageStateValue dbStorageState, + ApiUsageStateValue reExecState, ApiUsageStateValue jsExecState, ApiUsageStateValue tbelExecState, + ApiUsageStateValue emailExecState, ApiUsageStateValue smsExecState, ApiUsageStateValue alarmExecState) { + super(id, createdTime, tenantId); + this.entityId = (entityType != null && entityId != null) ? EntityIdFactory.getByTypeAndUuid(entityType, entityId) : null; + this.transportState = transportState; + this.dbStorageState = dbStorageState; + this.reExecState = reExecState; + this.jsExecState = jsExecState; + this.tbelExecState = tbelExecState; + this.emailExecState = emailExecState; + this.smsExecState = smsExecState; + this.alarmExecState = alarmExecState; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java new file mode 100644 index 0000000000..c5845efb87 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetFields extends AbstractEntityFields implements ProfileAwareFields { + + private String type; + private UUID assetProfileId; + private String label; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return assetProfileId; + } + + public AssetFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, + Long version, String type, String label, UUID assetProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.assetProfileId = assetProfileId; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java new file mode 100644 index 0000000000..bbe7efb56a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/AssetProfileFields.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class AssetProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public AssetProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java new file mode 100644 index 0000000000..00ab3776f8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/CustomerFields.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class CustomerFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + + public CustomerFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo, + String country, String state, String city, String address, String address2, String zip, String phone, String email) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java new file mode 100644 index 0000000000..af1640b7be --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DashboardFields.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + + +@Data +@NoArgsConstructor +@SuperBuilder +public class DashboardFields extends AbstractEntityFields { + + public DashboardFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version) { + super(id, createdTime, tenantId, customerId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java new file mode 100644 index 0000000000..f1622b83d9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceFields.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceFields extends AbstractEntityFields implements ProfileAwareFields { + + private String label; + private String type; + private UUID deviceProfileId; + private String additionalInfo; + + @JsonIgnore + @Override + public String getProfileName() { + return type; + } + + @JsonIgnore + @Override + public UUID getProfileId() { + return deviceProfileId; + } + + public DeviceFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, String type, + String label, UUID deviceProfileId, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.label = label; + this.type = type; + this.deviceProfileId = deviceProfileId; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java new file mode 100644 index 0000000000..5015196f69 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/DeviceProfileFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.DeviceProfileType; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class DeviceProfileFields extends AbstractEntityFields { + + private String type; + private boolean isDefault; + + public DeviceProfileFields(UUID id, long createdTime, UUID tenantId, String name, Long version, DeviceProfileType type, boolean isDefault) { + super(id, createdTime, tenantId, null, name, version); + this.type = type.name(); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java new file mode 100644 index 0000000000..460a43e7c8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EdgeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EdgeFields extends AbstractEntityFields { + + private String type; + private String label; + private String additionalInfo; + + public EdgeFields(UUID id, long createdTime, UUID tenantId, UUID customerId, String name, Long version, + String type, String label, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, name, version); + this.type = type; + this.label = label; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java new file mode 100644 index 0000000000..7536586f57 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -0,0 +1,171 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + +public interface EntityFields { + + Logger log = LoggerFactory.getLogger(EntityFields.class); + + default UUID getId() { + return null; + } + + default UUID getTenantId() { + return null; + } + + default UUID getCustomerId() { + return null; + } + + default long getCreatedTime() { + return 0; + } + + default String getName() { + return ""; + } + + default String getType() { + return ""; + } + + default String getLabel() { + return ""; + } + + default String getAdditionalInfo() { + return ""; + } + + default String getEmail() { + return ""; + } + + default String getCountry() { + return ""; + } + + default String getState() { + return ""; + } + + default String getCity() { + return ""; + } + + default String getAddress() { + return ""; + } + + default String getAddress2() { + return ""; + } + + default String getZip() { + return ""; + } + + default String getPhone() { + return ""; + } + + default String getRegion() { + return ""; + } + + default String getFirstName() { + return ""; + } + + default String getLastName() { + return ""; + } + + default boolean isEdgeTemplate() { + return false; + } + + default String getConfiguration() { + return ""; + } + + default String getSchedule() { + return ""; + } + + default EntityId getOriginatorId() { + return null; + } + + default String getQueueName() { + return ""; + } + + default String getServiceId() { + return ""; + } + + default boolean isDefault() { + return false; + } + + default UUID getOwnerId() { + return null; + } + + default Long getVersion() { + return null; + } + + default String getAsString(String key) { + return switch (key) { + case "createdTime" -> Long.toString(getCreatedTime()); + case "type" -> getType(); + case "label" -> getLabel(); + case "additionalInfo" -> getAdditionalInfo(); + case "email" -> getEmail(); + case "country" -> getCountry(); + case "state" -> getState(); + case "city" -> getCity(); + case "address" -> getAddress(); + case "address2" -> getAddress2(); + case "zip" -> getZip(); + case "phone" -> getPhone(); + case "region" -> getRegion(); + case "firstName" -> getFirstName(); + case "lastName" -> getLastName(); + case "edgeTemplate" -> Boolean.toString(isEdgeTemplate()); + case "configuration" -> getConfiguration(); + case "schedule" -> getSchedule(); + case "originatorId" -> getOriginatorId().getId().toString(); + case "originatorType" -> getOriginatorId().getEntityType().toString(); + case "queueName" -> getQueueName(); + case "serviceId" -> getServiceId(); + default -> { + log.warn("Unknown field '{}'", key); + yield null; + } + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java new file mode 100644 index 0000000000..aabac7865f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityIdFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityIdFields implements EntityFields { + + private UUID id; + private Long version; + + public EntityIdFields(UUID id, Long version) { + this.id = id; + this.version = version; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java new file mode 100644 index 0000000000..5635566cc4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityViewFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +@Data +@NoArgsConstructor +@SuperBuilder +public class EntityViewFields extends AbstractEntityFields { + + private String type; + private String additionalInfo; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java new file mode 100644 index 0000000000..5a0db392ef --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -0,0 +1,298 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.queue.QueueStats; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.UUID; + +public class FieldsUtil { + + public static EntityFields toFields(Object entity) { + if (entity instanceof Customer customer) { + return toFields(customer); + } else if (entity instanceof Tenant tenant) { + return toFields(tenant); + } else if (entity instanceof TenantProfile tenantProfile) { + return toFields(tenantProfile); + } else if (entity instanceof Device device) { + return toFields(device); + } else if (entity instanceof Asset asset) { + return toFields(asset); + } else if (entity instanceof Edge edge) { + return toFields(edge); + } else if (entity instanceof EntityView entityView) { + return toFields(entityView); + } else if (entity instanceof User user) { + return toFields(user); + } else if (entity instanceof Dashboard dashboard) { + return toFields(dashboard); + } else if (entity instanceof RuleChain ruleChain) { + return toFields(ruleChain); + } else if (entity instanceof RuleNode ruleNode) { + return toFields(ruleNode); + } else if (entity instanceof WidgetType widgetType) { + return toFields(widgetType); + } else if (entity instanceof WidgetsBundle widgetsBundle) { + return toFields(widgetsBundle); + } else if (entity instanceof DeviceProfile deviceProfile) { + return toFields(deviceProfile); + } else if (entity instanceof AssetProfile assetProfile) { + return toFields(assetProfile); + } else if (entity instanceof QueueStats queueStats) { + return toFields(queueStats); + } else if (entity instanceof ApiUsageState apiUsageState) { + return toFields(apiUsageState); + } else { + throw new IllegalArgumentException("Unsupported entity type: " + entity.getClass().getName()); + } + } + + private static CustomerFields toFields(Customer entity) { + return CustomerFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .version(entity.getVersion()) + .build(); + } + + private static TenantFields toFields(Tenant entity) { + return TenantFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getTitle()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .email(entity.getEmail()) + .country(entity.getCountry()) + .state(entity.getState()) + .city(entity.getCity()) + .address(entity.getAddress()) + .address2(entity.getAddress2()) + .zip(entity.getZip()) + .phone(entity.getPhone()) + .region(entity.getRegion()) + .version(entity.getVersion()) + .build(); + } + + private static TenantProfileFields toFields(TenantProfile tenantProfile) { + return TenantProfileFields.builder() + .id(tenantProfile.getUuidId()) + .createdTime(tenantProfile.getCreatedTime()) + .name(tenantProfile.getName()) + .isDefault(tenantProfile.isDefault()) + .build(); + } + + private static DeviceFields toFields(Device entity) { + return DeviceFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .deviceProfileId(entity.getDeviceProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static AssetFields toFields(Asset entity) { + return AssetFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .assetProfileId(entity.getAssetProfileId().getId()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EdgeFields toFields(Edge entity) { + return EdgeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .label(entity.getLabel()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static EntityViewFields toFields(EntityView entity) { + return EntityViewFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getName()) + .type(entity.getType()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static UserFields toFields(User entity) { + return UserFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .firstName(entity.getFirstName()) + .lastName(entity.getLastName()) + .email(entity.getEmail()) + .phone(entity.getPhone()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static DashboardFields toFields(Dashboard entity) { + return DashboardFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .customerId(getCustomerId(entity.getCustomerId())) + .name(entity.getTitle()) + .version(entity.getVersion()) + .build(); + } + + private static RuleChainFields toFields(RuleChain entity) { + return RuleChainFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .version(entity.getVersion()) + .build(); + } + + private static RuleNodeFields toFields(RuleNode entity) { + return RuleNodeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .additionalInfo(getText(entity.getAdditionalInfo())) + .build(); + } + + private static WidgetTypeFields toFields(WidgetType entity) { + return WidgetTypeFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static WidgetsBundleFields toFields(WidgetsBundle entity) { + return WidgetsBundleFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .version(entity.getVersion()) + .build(); + } + + private static AssetProfileFields toFields(DeviceProfile entity) { + return AssetProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static DeviceProfileFields toFields(AssetProfile entity) { + return DeviceProfileFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .name(entity.getName()) + .type(DeviceProfileType.DEFAULT.name()) + .isDefault(entity.isDefault()) + .version(entity.getVersion()) + .build(); + } + + private static QueueStatsFields toFields(QueueStats entity) { + return QueueStatsFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .queueName(entity.getQueueName()) + .serviceId(entity.getServiceId()) + .build(); + } + + private static ApiUsageStateFields toFields(ApiUsageState entity) { + return ApiUsageStateFields.builder() + .id(entity.getUuidId()) + .createdTime(entity.getCreatedTime()) + .entityId(entity.getEntityId()) + .transportState(entity.getTransportState()) + .dbStorageState(entity.getDbStorageState()) + .reExecState(entity.getReExecState()) + .jsExecState(entity.getJsExecState()) + .tbelExecState(entity.getTbelExecState()) + .emailExecState(entity.getEmailExecState()) + .smsExecState(entity.getSmsExecState()) + .alarmExecState(entity.getAlarmExecState()) + .build(); + } + + public static String getText(JsonNode node) { + return node != null ? node.asText() : ""; + } + + private static UUID getCustomerId(CustomerId customerId) { + return (customerId != null && !customerId.getId().equals(CustomerId.NULL_UUID)) ? customerId.getId() : null; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java new file mode 100644 index 0000000000..ff3af4ee65 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/GenericFields.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class GenericFields extends AbstractEntityFields { + + private String additionalInfo; + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } + + public GenericFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java new file mode 100644 index 0000000000..83134af716 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/ProfileAwareFields.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import java.util.UUID; + +public interface ProfileAwareFields extends EntityFields { + + String getProfileName(); + + UUID getProfileId(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java new file mode 100644 index 0000000000..32d74a4b39 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/QueueStatsFields.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class QueueStatsFields extends AbstractEntityFields { + + private String queueName; + private String serviceId; + + @Override + public String getName() { + return queueName + '_' + serviceId; + } + + public QueueStatsFields(UUID id, long createdTime, UUID tenantId, String queueName, String serviceId) { + super(id, createdTime, tenantId); + this.queueName = queueName; + this.serviceId = serviceId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java new file mode 100644 index 0000000000..ced202eeea --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleChainFields.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleChainFields extends AbstractEntityFields { + + private String additionalInfo; + + public RuleChainFields(UUID id, long createdTime, UUID tenantId, String name, Long version, JsonNode additionalInfo) { + super(id, createdTime, tenantId, name, version); + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java new file mode 100644 index 0000000000..82495a96de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/RuleNodeFields.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class RuleNodeFields implements EntityFields { + + private UUID id; + private long createdTime; + private String name; + private String additionalInfo; + + public RuleNodeFields(UUID id, long createdTime, String name, JsonNode additionalInfo) { + this.id = id; + this.createdTime = createdTime; + this.name = name; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java new file mode 100644 index 0000000000..6942d5ea7b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantFields extends AbstractEntityFields { + + private String additionalInfo; + private String country; + private String state; + private String city; + private String address; + private String address2; + private String zip; + private String phone; + private String email; + private String region; + + public TenantFields(UUID id, long createdTime, String name, Long version, + JsonNode additionalInfo, String country, String state, String city, String address, + String address2, String zip, String phone, String email, String region) { + super(id, createdTime, name, version); + this.additionalInfo = getText(additionalInfo); + this.country = country; + this.state = state; + this.city = city; + this.address = address; + this.address2 = address2; + this.zip = zip; + this.phone = phone; + this.email = email; + this.region = region; + } + + public TenantFields(UUID id, Long version) { + super(id, 0L, null, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java new file mode 100644 index 0000000000..766a160fc2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantProfileFields.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +@NoArgsConstructor +@SuperBuilder +public class TenantProfileFields extends AbstractEntityFields { + + private boolean isDefault; + + public TenantProfileFields(UUID id, long createdTime, String name, boolean isDefault) { + super(id, createdTime, TenantId.SYS_TENANT_ID.getId(), null, name, 0L); + this.isDefault = isDefault; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java new file mode 100644 index 0000000000..98e0efecc2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +import static org.thingsboard.server.common.data.edqs.fields.FieldsUtil.getText; + +@Data +@NoArgsConstructor +@SuperBuilder +public class UserFields extends AbstractEntityFields { + + private String firstName; + private String lastName; + private String email; + private String phone; + private String additionalInfo; + + public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId, + Long version, String firstName, String lastName, String email, + String phone, JsonNode additionalInfo) { + super(id, createdTime, tenantId, customerId, version); + this.firstName = firstName; + this.lastName = lastName; + this.email = email; + this.phone = phone; + this.additionalInfo = getText(additionalInfo); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java new file mode 100644 index 0000000000..330199d8b2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetTypeFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetTypeFields extends AbstractEntityFields { + + public WidgetTypeFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java new file mode 100644 index 0000000000..88d9a1a96c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/WidgetsBundleFields.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.fields; + +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +import java.util.UUID; + +@NoArgsConstructor +@SuperBuilder +public class WidgetsBundleFields extends AbstractEntityFields { + + public WidgetsBundleFields(UUID id, long createdTime, UUID tenantId, String name, Long version) { + super(id, createdTime, tenantId, name, version); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java new file mode 100644 index 0000000000..135061a842 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIncludeProperties; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EdqsRequest { + + private EntityDataQuery entityDataQuery; + private EntityCountQuery entityCountQuery; + @JsonIncludeProperties({"genericPermissions", "groupPermissions"}) + private MergedUserPermissions userPermissions; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java new file mode 100644 index 0000000000..bcb242e083 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/EdqsResponse.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class EdqsResponse { + + private PageData entityDataQueryResult; + private Long entityCountQueryResult; + private String error; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java new file mode 100644 index 0000000000..0c45812690 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/query/QueryResult.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.edqs.query; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.TsValue; + +import java.util.Collections; +import java.util.Map; + +@Data +@RequiredArgsConstructor +public class QueryResult { + + private final EntityId entityId; + private final boolean readAttrs; + private final boolean readTs; + private final Map> latest; + + public EntityData toOldEntityData() { + return new EntityData(entityId, readAttrs, readTs, latest, Collections.emptyMap(), Collections.emptyMap()); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java index 1b462db4da..cd1fdf9aa6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/UserAuthSettingsId.java @@ -15,11 +15,15 @@ */ package org.thingsboard.server.common.data.id; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.UUID; public class UserAuthSettingsId extends UUIDBased { - public UserAuthSettingsId(UUID id) { + @JsonCreator + public UserAuthSettingsId(@JsonProperty("id") UUID id) { super(id); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java index 8d46a46781..625a1d417c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/BasePageDataIterable.java @@ -23,6 +23,7 @@ import java.util.NoSuchElementException; public abstract class BasePageDataIterable implements Iterable, Iterator { private final int fetchSize; + private SortOrder sortOrder; private List currentItems; private int currentIdx; @@ -35,6 +36,12 @@ public abstract class BasePageDataIterable implements Iterable, Iterator iterator() { return this; @@ -43,7 +50,7 @@ public abstract class BasePageDataIterable implements Iterable, Iterator extends BasePageDataIterable { this.function = function; } + public PageDataIterable(FetchFunction function, int fetchSize, SortOrder sortOrder) { + super(fetchSize, sortOrder); + this.function = function; + } + @Override PageData fetchPageData(PageLink link) { return function.fetch(link); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java similarity index 77% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java rename to common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java index 579c5bfa92..b377d63ee4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QuerySecurityContext.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/permission/QueryContext.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.sql.query; +package org.thingsboard.server.common.data.permission; import lombok.AllArgsConstructor; import lombok.Getter; @@ -21,8 +21,12 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + @AllArgsConstructor -public class QuerySecurityContext { +public class QueryContext { @Getter private final TenantId tenantId; @@ -33,7 +37,11 @@ public class QuerySecurityContext { @Getter private final boolean ignorePermissionCheck; - public QuerySecurityContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { + @Getter + private final Map relatedParentIdMap = new HashMap<>(); + + public QueryContext(TenantId tenantId, CustomerId customerId, EntityType entityType) { this(tenantId, customerId, entityType, false); } + } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 473fc867fa..0b9deebfd9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.common.data.query; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.validation.NoXss; @@ -26,7 +25,6 @@ import java.io.Serializable; @RequiredArgsConstructor public class DynamicValue implements Serializable { - @JsonIgnore private T resolvedValue; private final DynamicValueSourceType sourceType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index 3dd24147e7..8c5d35f03d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.query; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Getter; import lombok.ToString; @@ -24,6 +25,7 @@ import java.util.List; @Schema @ToString +@JsonIgnoreProperties(ignoreUnknown = true) public class EntityCountQuery { @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java index 79730dc23a..fe9f0d3f68 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityData.java @@ -16,20 +16,22 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; -import lombok.RequiredArgsConstructor; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; import java.util.Map; @Data -@RequiredArgsConstructor +@AllArgsConstructor +@NoArgsConstructor public class EntityData { - private final EntityId entityId; - private final Map> latest; - private final Map timeseries; - private final Map aggLatest; + private EntityId entityId; + private Map> latest; + private Map timeseries; + private Map aggLatest; public EntityData(EntityId entityId, Map> latest, Map timeseries) { this(entityId, latest, timeseries, null); @@ -44,4 +46,5 @@ public class EntityData { aggLatest.clear(); } } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java index 383c613a38..17a712d838 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueConfig.java @@ -15,10 +15,22 @@ */ package org.thingsboard.server.common.data.queue; +import lombok.Data; + public interface QueueConfig { boolean isConsumerPerPartition(); int getPollInterval(); + static QueueConfig of(boolean consumerPerPartition, long pollInterval) { + return new BasicQueueConfig(consumerPerPartition, (int) pollInterval); + } + + @Data + class BasicQueueConfig implements QueueConfig { + private final boolean consumerPerPartition; + private final int pollInterval; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java index 6c648daf02..9c3536dea1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java @@ -18,13 +18,15 @@ package org.thingsboard.server.common.data.queue; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasEntityType; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; @EqualsAndHashCode(callSuper = true) @Data -public class QueueStats extends BaseData implements HasTenantId { +public class QueueStats extends BaseData implements HasTenantId, HasEntityType { private TenantId tenantId; private String queueName; private String serviceId; @@ -36,4 +38,8 @@ public class QueueStats extends BaseData implements HasTenantId { super(id); } + @Override + public EntityType getEntityType() { + return EntityType.QUEUE_STATS; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java index 62b5caafd3..927d592cd7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/relation/EntityRelation.java @@ -25,6 +25,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasVersion; +import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.validation.Length; @@ -34,7 +35,7 @@ import java.io.Serializable; @Schema @EqualsAndHashCode(exclude = "additionalInfoBytes") @ToString(exclude = {"additionalInfoBytes"}) -public class EntityRelation implements HasVersion, Serializable { +public class EntityRelation implements HasVersion, Serializable, EdqsObject { private static final long serialVersionUID = 2807343040519543363L; @@ -107,7 +108,7 @@ public class EntityRelation implements HasVersion, Serializable { return typeGroup; } - @Schema(description = "Additional parameters of the relation",implementation = com.fasterxml.jackson.databind.JsonNode.class) + @Schema(description = "Additional parameters of the relation", implementation = com.fasterxml.jackson.databind.JsonNode.class) public JsonNode getAdditionalInfo() { return BaseDataWithAdditionalInfo.getJson(() -> additionalInfo, () -> additionalInfoBytes); } @@ -116,4 +117,14 @@ public class EntityRelation implements HasVersion, Serializable { BaseDataWithAdditionalInfo.setJson(addInfo, json -> this.additionalInfo = json, bytes -> this.additionalInfoBytes = bytes); } + @JsonIgnore + public String key() { + return "r_" + from + "_" + to + "_" + typeGroup + "_" + type; + } + + @Override + public Long version() { + return version; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index e89f3c1ad8..2fc3f4c040 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.util; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -75,4 +76,11 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static HashSet concat(Set set1, Set set2) { + HashSet result = new HashSet<>(); + result.addAll(set1); + result.addAll(set2); + return result; + } + } diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml new file mode 100644 index 0000000000..40e89acee6 --- /dev/null +++ b/common/edqs/pom.xml @@ -0,0 +1,97 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0PE-SNAPSHOT + common + + org.thingsboard.common + edqs + jar + + Thingsboard Server EDQS API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.rocksdb + rocksdbjni + + + org.thingsboard.common + proto + + + org.thingsboard.common + data + + + org.thingsboard.common + util + + + org.thingsboard.common + message + + + org.thingsboard.common + stats + + + org.thingsboard.common + cluster-api + + + org.thingsboard.common + queue + + + org.apache.kafka + kafka-clients + + + com.github.ben-manes.caffeine + caffeine + + + org.springframework + spring-context-support + + + org.springframework.boot + spring-boot-autoconfigure + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java new file mode 100644 index 0000000000..c1ce21507e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ApiUsageStateData.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class ApiUsageStateData extends BaseEntityData { + + public ApiUsageStateData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.API_USAGE_STATE; + } + + @Override + public String getEntityName() { + return getEntityOwnerName(); + } + + @Override + public String getEntityOwnerName() { + return repo.getOwnerName(fields.getEntityId()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java new file mode 100644 index 0000000000..ce2a63a2ec --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/AssetData.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class AssetData extends ProfileAwareData { + + public AssetData(UUID id) { + super(id); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java new file mode 100644 index 0000000000..11c8b269b9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -0,0 +1,180 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString +public abstract class BaseEntityData implements EntityData { + + @Getter + private final UUID id; + @Getter + protected final Map serverAttrMap; + @Getter + private final Map tMap; + + @Getter + @Setter + private volatile UUID customerId; + + @Setter + protected TenantRepo repo; + + @Getter + @Setter + protected volatile T fields; + + public BaseEntityData(UUID id) { + this.id = id; + this.serverAttrMap = new ConcurrentHashMap<>(); + this.tMap = new ConcurrentHashMap<>(); + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE, SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + default -> null; + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return serverAttrMap.put(keyId, value) == null; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return serverAttrMap.remove(keyId) != null; + } + + @Override + public DataPoint getTs(Integer keyId) { + return tMap.get(keyId); + } + + @Override + public boolean putTs(Integer keyId, DataPoint value) { + return tMap.put(keyId, value) == null; + } + + @Override + public boolean removeTs(Integer keyId) { + return tMap.remove(keyId) != null; + } + + @Override + public EntityType getOwnerType() { + return customerId != null ? EntityType.CUSTOMER : EntityType.TENANT; + } + + @Override + public DataPoint getDataPoint(DataKey key, QueryContext ctx) { + return switch (key.type()) { + case TIME_SERIES -> getTs(key.keyId()); + case ATTRIBUTE, SERVER_ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE -> getAttr(key.keyId(), key.type()); + case ENTITY_FIELD -> getField(key, ctx); + default -> throw new RuntimeException(key.type() + " not supported"); + }; + } + + private DataPoint getField(DataKey newKey, QueryContext ctx) { + if (fields == null) { + return null; + } + String key = newKey.key(); + return switch (key) { + case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); + case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); + }; + } + + @Override + public String getField(String name) { + if (fields == null) { + return null; + } + return switch (name) { + case "name" -> getEntityName(); + case "ownerName" -> getEntityOwnerName(); + case "ownerType" -> customerId != null ? EntityType.CUSTOMER.name() : EntityType.TENANT.name(); + case "entityType" -> Optional.ofNullable(getEntityType()).map(EntityType::name).orElse(""); + default -> fields.getAsString(name); + }; + } + + public String getEntityOwnerName() { + return repo.getOwnerName(getCustomerId() == null || CustomerId.NULL_UUID.equals(getCustomerId()) ? null : + new CustomerId(getCustomerId())); + } + + public String getEntityName() { + return getFields().getName(); + } + + private String getRelatedParentId(QueryContext ctx) { + return Optional.ofNullable(ctx.getRelatedParentIdMap().get(getId())) + .map(UUID::toString) + .orElse(""); + } + + @Override + public EntityType getEntityType() { + return null; + } + + @Override + public boolean isEmpty() { + return fields == null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BaseEntityData that = (BaseEntityData) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java new file mode 100644 index 0000000000..15e2573b81 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/CustomerData.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; + +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class CustomerData extends BaseEntityData { + + private final ConcurrentMap>> entitiesById = new ConcurrentHashMap<>(); + + public CustomerData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + + public Collection> getEntities(EntityType entityType) { + var map = entitiesById.get(entityType); + if (map == null) { + return Collections.emptyList(); + } else { + return map.values(); + } + } + + public void addOrUpdate(EntityData ed) { + entitiesById.computeIfAbsent(ed.getEntityType(), et -> new ConcurrentHashMap<>()).put(ed.getId(), ed); + } + + public boolean remove(EntityData ed) { + var map = entitiesById.get(ed.getEntityType()); + if (map != null) { + return map.remove(ed.getId()) != null; + } else { + return false; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java new file mode 100644 index 0000000000..42042b2a9f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/DeviceData.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.DataPoint; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +@ToString(callSuper = true) +public class DeviceData extends ProfileAwareData { + + private final Map clientAttrMap; + private final Map sharedAttrMap; + + public DeviceData(UUID entityId) { + super(entityId); + this.clientAttrMap = new ConcurrentHashMap<>(); + this.sharedAttrMap = new ConcurrentHashMap<>(); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + public DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType) { + return switch (entityKeyType) { + case ATTRIBUTE -> getAttributeDataPoint(keyId); + case SERVER_ATTRIBUTE -> serverAttrMap.get(keyId); + case CLIENT_ATTRIBUTE -> clientAttrMap.get(keyId); + case SHARED_ATTRIBUTE -> sharedAttrMap.get(keyId); + default -> throw new RuntimeException(entityKeyType + " not implemented"); + }; + } + + @Override + public boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.put(keyId, value) == null; + case CLIENT_SCOPE -> clientAttrMap.put(keyId, value) == null; + case SHARED_SCOPE -> sharedAttrMap.put(keyId, value) == null; + }; + } + + @Override + public boolean removeAttr(Integer keyId, AttributeScope scope) { + return switch (scope) { + case SERVER_SCOPE -> serverAttrMap.remove(keyId) != null; + case CLIENT_SCOPE -> clientAttrMap.remove(keyId) != null; + case SHARED_SCOPE -> sharedAttrMap.remove(keyId) != null; + }; + } + + private DataPoint getAttributeDataPoint(Integer keyId) { + DataPoint dp = serverAttrMap.get(keyId); + if (dp == null) { + dp = sharedAttrMap.get(keyId); + if (dp == null) { + dp = clientAttrMap.get(keyId); + } + } + return dp; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java new file mode 100644 index 0000000000..c373baf305 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityData.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.UUID; + +public interface EntityData { + + UUID getId(); + + EntityType getEntityType(); + + UUID getCustomerId(); + + void setCustomerId(UUID customerId); + + void setRepo(TenantRepo repo); + + T getFields(); + + void setFields(T fields); + + DataPoint getAttr(Integer keyId, EntityKeyType entityKeyType); + + boolean putAttr(Integer keyId, AttributeScope scope, DataPoint value); + + boolean removeAttr(Integer keyId, AttributeScope scope); + + DataPoint getTs(Integer keyId); + + boolean putTs(Integer keyId, DataPoint value); + + boolean removeTs(Integer keyId); + + EntityType getOwnerType(); + + DataPoint getDataPoint(DataKey key, QueryContext queryContext); + + String getField(String name); + + boolean isEmpty(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java new file mode 100644 index 0000000000..d956519cc4 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityGroupData.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; + +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class EntityGroupData extends BaseEntityData { + + private final ConcurrentMap> entitiesById = new ConcurrentHashMap<>(); + + public EntityGroupData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_GROUP; + } + + public Collection> getEntities() { + return entitiesById.values(); + } + + public boolean addOrUpdate(EntityData ed) { + return entitiesById.put(ed.getId(), ed) == null; + } + + public boolean remove(EntityData ed) { + return entitiesById.remove(ed.getId()) != null; + } + + public EntityData getEntity(UUID entityId) { + return entitiesById.get(entityId); + } + + public boolean remove(UUID toId) { + return entitiesById.remove(toId) != null; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java new file mode 100644 index 0000000000..6057ac17e7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/EntityProfileData.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class EntityProfileData extends BaseEntityData { + + private final EntityType entityType; + + public EntityProfileData(UUID entityId, EntityType entityType) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java new file mode 100644 index 0000000000..8fd8924b5c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/GenericData.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; + +import java.util.UUID; + +@ToString(callSuper = true) +public class GenericData extends BaseEntityData { + + private final EntityType entityType; + + public GenericData(EntityType entityType, UUID entityId) { + super(entityId); + this.entityType = entityType; + } + + @Override + public EntityType getEntityType() { + return entityType; + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java new file mode 100644 index 0000000000..f98c8e6d9b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/ProfileAwareData.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.edqs.fields.ProfileAwareFields; + +import java.util.UUID; + +public abstract class ProfileAwareData extends BaseEntityData { + + public ProfileAwareData(UUID id) { + super(id); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java new file mode 100644 index 0000000000..3a660af813 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationData.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; + +import java.util.UUID; + +public record RelationData(UUID fromId, EntityType fromType, UUID toId, EntityType toType, String type, + RelationTypeGroup typeGroup) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java new file mode 100644 index 0000000000..e479c94d00 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationInfo.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.Data; + +@Data +public class RelationInfo { + + private final String type; + private final EntityData target; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java new file mode 100644 index 0000000000..92cf10e1d0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/RelationsRepo.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@NoArgsConstructor +public class RelationsRepo { + + private final ConcurrentMap> fromRelations = new ConcurrentHashMap<>(); + private final ConcurrentMap> toRelations = new ConcurrentHashMap<>(); + + public boolean add(EntityData from, EntityData to, String type) { + boolean addedFromRelation = fromRelations.computeIfAbsent(from.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, to)); + boolean addedToRelation = toRelations.computeIfAbsent(to.getId(), k -> ConcurrentHashMap.newKeySet()).add(new RelationInfo(type, from)); + return addedFromRelation || addedToRelation; + } + + public Set getFrom(UUID entityId) { + var result = fromRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public Set getTo(UUID entityId) { + var result = toRelations.get(entityId); + return result == null ? Collections.emptySet() : result; + } + + public boolean remove(UUID from, UUID to, String type) { + boolean removedFromRelation = false; + boolean removedToRelation = false; + Set fromRelations = this.fromRelations.get(from); + if (fromRelations != null) { + removedFromRelation = fromRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(to) && relationInfo.getType().equals(type)); + } + Set toRelations = this.toRelations.get(to); + if (toRelations != null) { + removedToRelation = toRelations.removeIf(relationInfo -> relationInfo.getTarget().getId().equals(from) && relationInfo.getType().equals(type)); + } + return removedFromRelation || removedToRelation; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java new file mode 100644 index 0000000000..a9034e251c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/TenantData.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; + +import java.util.UUID; + +public class TenantData extends BaseEntityData { + + public TenantData(UUID entityId) { + super(entityId); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java new file mode 100644 index 0000000000..5eb16f72fa --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/AbstractDataPoint.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public abstract class AbstractDataPoint implements DataPoint { + + @Getter + private final long ts; + + @Override + public String getStr() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public long getLong() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public double getDouble() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public boolean getBool() { + throw new RuntimeException(NOT_SUPPORTED); + } + + @Override + public String getJson() { + throw new RuntimeException(NOT_SUPPORTED); + } + + public String toString() { + return valueToString(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java new file mode 100644 index 0000000000..ccc97a2f84 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/BoolDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class BoolDataPoint extends AbstractDataPoint { + + @Getter + private final boolean value; + + public BoolDataPoint(long ts, boolean value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.BOOLEAN; + } + + @Override + public boolean getBool() { + return value; + } + + @Override + public String valueToString() { + return Boolean.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java new file mode 100644 index 0000000000..b10f3badf0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public class CompressedJsonDataPoint extends CompressedStringDataPoint { + + public CompressedJsonDataPoint(long ts, String value) { + super(ts, value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java new file mode 100644 index 0000000000..b679c94f4e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbBytePool; +import org.xerial.snappy.Snappy; + +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class CompressedStringDataPoint extends AbstractDataPoint { + + public static final int MIN_STR_SIZE_TO_COMPRESS = 512; + @Getter + private final byte[] value; + + public static final AtomicInteger cnt = new AtomicInteger(); + public static final AtomicLong uncompressedLength = new AtomicLong(); + public static final AtomicLong compressedLength = new AtomicLong(); + + @SneakyThrows + public CompressedStringDataPoint(long ts, String value) { + super(ts); + cnt.incrementAndGet(); + uncompressedLength.addAndGet(value.getBytes(StandardCharsets.UTF_8).length); + this.value = TbBytePool.intern(Snappy.compress(value)); + compressedLength.addAndGet(this.value.length); + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @SneakyThrows + @Override + public String getStr() { + return Snappy.uncompressString(value); + } + + @SneakyThrows + @Override + public String valueToString() { + return Snappy.uncompressString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java new file mode 100644 index 0000000000..3871bca114 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DataPoint.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import org.thingsboard.server.common.data.kv.DataType; + +public interface DataPoint { + + String NOT_SUPPORTED = "Not supported!"; + + long getTs(); + + DataType getType(); + + String getStr(); + + long getLong(); + + double getDouble(); + + boolean getBool(); + + String getJson(); + + String valueToString(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java new file mode 100644 index 0000000000..a7dcdf9e82 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/DoubleDataPoint.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class DoubleDataPoint extends AbstractDataPoint { + + @Getter + private final double value; + + public DoubleDataPoint(long ts, double value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.DOUBLE; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Double.toString(value); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java new file mode 100644 index 0000000000..df318b6430 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbStringPool; + +public class JsonDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public JsonDataPoint(long ts, String value) { + super(ts); + this.value = TbStringPool.intern(value); + } + + @Override + public DataType getType() { + return DataType.JSON; + } + + @Override + public String getJson() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java new file mode 100644 index 0000000000..ac54f19707 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/LongDataPoint.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; + +public class LongDataPoint extends AbstractDataPoint { + + @Getter + private final long value; + + public LongDataPoint(long ts, long value) { + super(ts); + this.value = value; + } + + @Override + public DataType getType() { + return DataType.LONG; + } + + @Override + public long getLong() { + return value; + } + + @Override + public double getDouble() { + return value; + } + + @Override + public String valueToString() { + return Long.toString(value); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java new file mode 100644 index 0000000000..e1e2b5d0ef --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.data.dp; + +import lombok.Getter; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.edqs.repo.TbStringPool; + +public class StringDataPoint extends AbstractDataPoint { + + @Getter + private final String value; + + public StringDataPoint(long ts, String value) { + this(ts, value, true); + } + + public StringDataPoint(long ts, String value, boolean deduplicate) { + super(ts); + this.value = deduplicate ? TbStringPool.intern(value) : value; + } + + @Override + public DataType getType() { + return DataType.STRING; + } + + @Override + public String getStr() { + return value; + } + + @Override + public String valueToString() { + return value; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java new file mode 100644 index 0000000000..d59d0cc671 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/load/TenantRepoLoader.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.load; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.RandomStringUtils; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.edqs.processor.EdqsConverter; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +@RequiredArgsConstructor +public class TenantRepoLoader { + + private static final int DEVICE_COUNT = 100000; + private static final int ATTRS_PER_DEVICE = 30; + private static final int ATTRS_AVG_STR_LENGTH = 12; + private static final int ATTRS_AVG_JSON_LENGTH = 265; + private static final int TS_PER_DEVICE = 29; + private static final int TS_AVG_STR_LENGTH = 59; + private static final int TS_AVG_JSON_LENGTH = 4005; + + private static final Map ATTR_CHANCES = new HashMap<>(); + private static final Random random = new Random(); + + static { + ATTR_CHANCES.put(DataType.BOOLEAN, 5); + ATTR_CHANCES.put(DataType.STRING, 49); + ATTR_CHANCES.put(DataType.LONG, 34); + ATTR_CHANCES.put(DataType.DOUBLE, 2); + ATTR_CHANCES.put(DataType.JSON, 10); + } + + private static final Map TS_CHANCES = new HashMap<>(); + + static { + TS_CHANCES.put(DataType.BOOLEAN, 6); + TS_CHANCES.put(DataType.STRING, 19); + TS_CHANCES.put(DataType.LONG, 36); + TS_CHANCES.put(DataType.DOUBLE, 32); + TS_CHANCES.put(DataType.JSON, 7); + } + + + @Getter + private final TenantRepo tenantRepo; + + public void load() { + long ts = System.currentTimeMillis() - DEVICE_COUNT; + for (int i = 0; i < DEVICE_COUNT; i++) { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setCreatedTime(ts + i); + device.setName("Device " + i); + device.setLabel("Device Label" + i); + device.setType("Device Type " + (i % 100)); + tenantRepo.addOrUpdate(EdqsConverter.toEntity(EntityType.DEVICE, device)); + for (int j = 0; j < ATTRS_PER_DEVICE; j++) { + String key = getRandomKey(); + AttributeKv attributeKv = new AttributeKv(); + attributeKv.setEntityId(deviceId); + attributeKv.setScope(AttributeScope.SERVER_SCOPE); + attributeKv.setKey(key); + attributeKv.setLastUpdateTs(ts); + attributeKv.setValue(getRandomKvEntry(key, ATTR_CHANCES, ATTRS_AVG_STR_LENGTH, ATTRS_AVG_JSON_LENGTH)); + tenantRepo.addOrUpdateAttribute(attributeKv); + } + for (int j = 0; j < TS_PER_DEVICE; j++) { + String key = getRandomKey(); + LatestTsKv latestTsKv = new LatestTsKv(); + latestTsKv.setEntityId(deviceId); + latestTsKv.setKey(key); + latestTsKv.setTs(ts); + latestTsKv.setValue(getRandomKvEntry(key, TS_CHANCES, TS_AVG_STR_LENGTH, TS_AVG_JSON_LENGTH)); + tenantRepo.addOrUpdateLatestKv(latestTsKv); + } + } + } + + private KvEntry getRandomKvEntry(String key, Map chances, int strLength, int jsnLength) { + int i = random.nextInt(100); + int s = 0; + for (var pair : chances.entrySet()) { + s += pair.getValue(); + if (i < s) { + switch (pair.getKey()) { + case BOOLEAN -> { + return new BooleanDataEntry(key, random.nextBoolean()); + } + case LONG -> { + return new LongDataEntry(key, random.nextLong()); + } + case DOUBLE -> { + return new DoubleDataEntry(key, random.nextDouble()); + } + case STRING -> { + return new StringDataEntry(key, StringUtils.randomAlphanumeric(strLength)); + } + case JSON -> { + return new JsonDataEntry(key, StringUtils.randomAlphanumeric(jsnLength)); + } + } + } + } + throw new RuntimeException("Something went wrong"); + } + + private String getRandomKey() { + return RandomStringUtils.randomAlphabetic(10); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java new file mode 100644 index 0000000000..125f6c49fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsConverter.java @@ -0,0 +1,183 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.FieldsUtil; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +@Service +public class EdqsConverter { + + private final Map> converters = new HashMap<>(); + private final Converter defaultConverter = new JsonConverter<>(Entity.class); + + { + converters.put(ObjectType.RELATION, new JsonConverter<>(EntityRelation.class)); + converters.put(ObjectType.ATTRIBUTE_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, AttributeKv attributeKv) { + // TODO: some attributes may not fit into kafka + var proto = TransportProtos.AttributeKvProto.newBuilder() + .setEntityIdMSB(attributeKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(attributeKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(attributeKv.getEntityId().getEntityType())) + .setScope(TransportProtos.AttributeScopeProto.forNumber(attributeKv.getScope().ordinal())) + .setKey(attributeKv.getKey()) + .setVersion(attributeKv.getVersion()); + if (attributeKv.getLastUpdateTs() != null) { + proto.setLastUpdateTs(attributeKv.getLastUpdateTs()); + } + if (attributeKv.getValue() != null) { + proto.setValue(KvProtoUtil.toKeyValueTypeProto(attributeKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public AttributeKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.AttributeKvProto proto = TransportProtos.AttributeKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + AttributeScope scope = AttributeScope.values()[proto.getScope().getNumber()]; + KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + return AttributeKv.builder() + .entityId(entityId) + .scope(scope) + .key(proto.getKey()) + .version(proto.getVersion()) + .lastUpdateTs(proto.getLastUpdateTs()) + .value(value) + .build(); + } + }); + converters.put(ObjectType.LATEST_TS_KV, new Converter() { + @Override + public byte[] serialize(ObjectType type, LatestTsKv latestTsKv) { + var proto = TransportProtos.LatestTsKvProto.newBuilder() + .setEntityIdMSB(latestTsKv.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(latestTsKv.getEntityId().getId().getLeastSignificantBits()) + .setEntityType(ProtoUtils.toProto(latestTsKv.getEntityId().getEntityType())) + .setKey(latestTsKv.getKey()) + .setVersion(latestTsKv.getVersion()); + if (latestTsKv.getTs() != null) { + proto.setTs(latestTsKv.getTs()); + } + if (latestTsKv.getValue() != null) { + proto.setValue(KvProtoUtil.toKeyValueTypeProto(latestTsKv.getValue())); + } + return proto.build().toByteArray(); + } + + @Override + public LatestTsKv deserialize(ObjectType type, byte[] bytes) throws Exception { + TransportProtos.LatestTsKvProto proto = TransportProtos.LatestTsKvProto.parseFrom(bytes); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ProtoUtils.fromProto(proto.getEntityType()), + new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + KvEntry value = proto.hasValue() ? KvProtoUtil.fromTsKvProto(proto.getValue()) : null; + return LatestTsKv.builder() + .entityId(entityId) + .key(proto.getKey()) + .ts(proto.getTs()) + .version(proto.getVersion()) + .value(value) + .build(); + } + }); + } + + public static Entity toEntity(EntityType entityType, Object entity) { + Entity edqsEntity = new Entity(); + edqsEntity.setType(entityType); + edqsEntity.setFields(FieldsUtil.toFields(entity)); + return edqsEntity; + } + + public EdqsObject check(ObjectType type, Object object) { + if (object instanceof EdqsObject edqsObject) { + return edqsObject; + } else { + return toEntity(type.toEntityType(), object); + } + } + + @SuppressWarnings("unchecked") + @SneakyThrows + public byte[] serialize(ObjectType type, T value) { + Converter converter = (Converter) converters.get(type); + if (converter != null) { + return converter.serialize(type, value); + } else { + return defaultConverter.serialize(type, (Entity) value); + } + } + + @SneakyThrows + public EdqsObject deserialize(ObjectType type, byte[] bytes) { + Converter converter = converters.get(type); + if (converter != null) { + return converter.deserialize(type, bytes); + } else { + return defaultConverter.deserialize(type, bytes); + } + } + + @RequiredArgsConstructor + private static class JsonConverter implements Converter { + + private final Class type; + + @Override + public byte[] serialize(ObjectType objectType, T value) { + return JacksonUtil.writeValueAsBytes(value); + } + + @Override + public T deserialize(ObjectType objectType, byte[] bytes) { + return JacksonUtil.fromBytes(bytes, this.type); + } + + } + + private interface Converter { + + byte[] serialize(ObjectType type, T value) throws Exception; + + T deserialize(ObjectType type, byte[] bytes) throws Exception; + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java new file mode 100644 index 0000000000..5383311e1f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.state.EdqsStateService; +import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueHandler; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.edqs.EdqsComponent; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class EdqsProcessor implements TbQueueHandler, TbProtoQueueMsg> { + + private final EdqsQueueFactory queueFactory; + private final EdqsConverter converter; + private final EdqRepository repository; + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + @Autowired @Lazy + private EdqsStateService stateService; + + private MainQueueConsumerManager, QueueConfig> eventsConsumer; + private TbQueueResponseTemplate, TbProtoQueueMsg> responseTemplate; + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + private ListeningExecutorService requestExecutor; + + private final VersionsStore versionsStore = new VersionsStore(); + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-mgmt"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); + requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + + eventsConsumer = MainQueueConsumerManager., QueueConfig>builder() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) + .config(QueueConfig.of(true, config.getPollInterval())) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + process(msg, EdqsQueue.EVENTS); + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS)) + .consumerExecutor(consumersExecutor) + .taskExecutor(mgmtExecutor) + .scheduler(scheduler) + .build(); + responseTemplate = queueFactory.createEdqsResponseTemplate(); + } + + @AfterStartUp(order = 1) + public void start() { + responseTemplate.launch(this); + } + + @EventListener + public void onPartitionsChange(PartitionChangeEvent event) { + if (event.getServiceType() != ServiceType.EDQS) { + return; + } + consumersExecutor.submit(() -> { + try { + Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); + Set partitions = newPartitions.stream() + .map(tpi -> tpi.withUseInternalPartition(true)) + .collect(Collectors.toSet()); + + try { + stateService.restore(withTopic(partitions, EdqsQueue.STATE.getTopic())); // blocks until restored + } catch (Exception e) { + log.error("Failed to process restore for partitions {}", partitions, e); + } + eventsConsumer.update(withTopic(partitions, EdqsQueue.EVENTS.getTopic())); + responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); + + Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); + } catch (Throwable t) { + log.error("Failed to handle partition change event {}", event, t); + } + }); + } + + @Override + public ListenableFuture> handle(TbProtoQueueMsg queueMsg) { + ToEdqsMsg toEdqsMsg = queueMsg.getValue(); + return requestExecutor.submit(() -> { + EdqsResponse response = new EdqsResponse(); + try { + EdqsRequest request = JacksonUtil.fromString(toEdqsMsg.getRequestMsg().getValue(), EdqsRequest.class); + TenantId tenantId = getTenantId(toEdqsMsg); + CustomerId customerId = getCustomerId(toEdqsMsg); + log.info("[{}] Handling request: {}", tenantId, request); + + if (request.getEntityDataQuery() != null) { + PageData result = repository.findEntityDataByQuery(tenantId, customerId, + request.getUserPermissions(), request.getEntityDataQuery(), false); + response.setEntityDataQueryResult(result.mapData(QueryResult::toOldEntityData)); + } else if (request.getEntityCountQuery() != null) { + long result = repository.countEntitiesByQuery(tenantId, customerId, request.getUserPermissions(), request.getEntityCountQuery(), tenantId.isSysTenantId()); + response.setEntityCountQueryResult(result); + } + + log.info("Answering with response: {}", response); + } catch (Throwable e) { + response.setError(ExceptionUtils.getStackTrace(e)); // TODO: return only the message + log.info("Answering with error", e); + } + return new TbProtoQueueMsg<>(queueMsg.getKey(), FromEdqsMsg.newBuilder() + .setResponseMsg(TransportProtos.EdqsResponseMsg.newBuilder() + .setValue(JacksonUtil.toString(response)) + .build()) + .build(), queueMsg.getHeaders()); + }); + } + + public void process(ToEdqsMsg edqsMsg, EdqsQueue queue) { + if (edqsMsg.hasEventMsg()) { + EdqsEventMsg eventMsg = edqsMsg.getEventMsg(); + TenantId tenantId = getTenantId(edqsMsg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + String key = eventMsg.getKey(); + Long version = eventMsg.hasVersion() ? eventMsg.getVersion() : null; + + if (version != null) { + if (!versionsStore.isNew(key, version)) { + return; + } + } else { + log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, edqsMsg); + } + if (queue != EdqsQueue.STATE) { + stateService.save(tenantId, objectType, key, eventType, edqsMsg); + } + + EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); + log.info("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + + EdqsEvent event = EdqsEvent.builder() + .tenantId(tenantId) + .objectType(objectType) + .eventType(eventType) + .object(object) + .build(); + repository.processEvent(event); + } + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + private CustomerId getCustomerId(ToEdqsMsg edqsMsg) { + if (edqsMsg.getCustomerIdMSB() != 0 && edqsMsg.getCustomerIdLSB() != 0) { + return new CustomerId(new UUID(edqsMsg.getCustomerIdMSB(), edqsMsg.getCustomerIdLSB())); + } else { + return null; + } + } + + private Set withTopic(Set partitions, String topic) { + return partitions.stream() + .map(tpi -> tpi.withTopic(topic)) + .collect(Collectors.toSet()); + } + + @PreDestroy + public void destroy() throws InterruptedException { + eventsConsumer.stop(); + eventsConsumer.awaitStop(); + responseTemplate.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); + requestExecutor.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java new file mode 100644 index 0000000000..762e5b37ec --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.processor; + +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; + +@Slf4j +public class EdqsProducer { + + private final EdqsQueue queue; + private final EdqsPartitionService partitionService; + + private final TbQueueProducer> producer; + + @Builder + public EdqsProducer(EdqsQueue queue, + EdqsPartitionService partitionService, + TbQueueProducer> producer) { + this.queue = queue; + this.partitionService = partitionService; + this.producer = producer; + } + + // TODO: queue prefix! + + public void send(TenantId tenantId, ObjectType type, String key, ToEdqsMsg msg) { + String topic = queue.getTopic(); + TbQueueCallback callback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}][{}] Failed to publish msg to {}: {}", tenantId, type, key, topic, msg, t); + } + }; + if (producer instanceof TbKafkaProducerTemplate> kafkaProducer) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .partition(partitionService.resolvePartition(tenantId)) + .useInternalPartition(true) + .build(); + kafkaProducer.send(tpi, key, new TbProtoQueueMsg<>(null, msg), callback); // specifying custom key for compaction + } else { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(topic) + .build(); + producer.send(tpi, new TbProtoQueueMsg<>(null, msg), callback); + } + } + + public void stop() { + producer.stop(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java new file mode 100644 index 0000000000..0b0805639a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/DataKey.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyType; + +public record DataKey(EntityKeyType type, String key, Integer keyId) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java new file mode 100644 index 0000000000..6a7c1e0ce0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsCountQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +public class EdqsCountQuery extends EdqsQuery { + + @Builder + EdqsCountQuery(EntityFilter entityFilter, boolean hasKeyFilters, List keyFilters) { + super(entityFilter, hasKeyFilters, keyFilters); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java new file mode 100644 index 0000000000..7f70d2ac6c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsDataQuery.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.util.CollectionsUtil; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Getter +public class EdqsDataQuery extends EdqsQuery { + + private final int pageSize; + private final int page; + private final boolean hasTextSearch; + private final String textSearch; + private final boolean defaultSort; + private final DataKey sortKey; + private final EntityDataSortOrder.Direction sortDirection; + private final List entityFields; + private final List latestValues; + + @Builder + public EdqsDataQuery(EntityFilter entityFilter, List keyFilters, + int pageSize, int page, String textSearch, DataKey sortKey, EntityDataSortOrder.Direction sortDirection, + List entityFields, List latestValues) { + super(entityFilter, CollectionsUtil.isNotEmpty(keyFilters), keyFilters); + this.pageSize = pageSize; + this.page = page; + this.hasTextSearch = StringUtils.isNotBlank(textSearch); + this.textSearch = textSearch; + this.defaultSort = EntityKeyType.ENTITY_FIELD.equals(sortKey.type()) && "createdTime".equals(sortKey.key()) && EntityDataSortOrder.Direction.DESC.equals(sortDirection); + this.sortKey = sortKey; + this.sortDirection = sortDirection; + this.entityFields = entityFields; + this.latestValues = latestValues; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java new file mode 100644 index 0000000000..a728ea567e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsFilter.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; + +public record EdqsFilter(DataKey key, EntityKeyValueType valueType, KeyFilterPredicate predicate) { + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java new file mode 100644 index 0000000000..31e2c482b1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/EdqsQuery.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.query.EntityFilter; + +import java.util.List; + +@Data +public abstract class EdqsQuery { + + private final EntityFilter entityFilter; + private final boolean hasKeyFilters; + private final List keyFilters; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java new file mode 100644 index 0000000000..7185cfc63f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/SortableEntityData.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.edqs.data.EntityData; + +import java.util.UUID; + +@Data +public class SortableEntityData { + + private final EntityData entityData; + private String sortValue; + private boolean readAttrs; + private boolean readTs; + + public UUID getId(){ + return entityData.getId(); + } + + public EntityId getEntityId() { + return EntityIdFactory.getByTypeAndUuid(entityData.getEntityType(), entityData.getId()); + } +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java new file mode 100644 index 0000000000..87df5586c1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityGroupQueryProcessor.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public abstract class AbstractEntityGroupQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public AbstractEntityGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions); + Map mergedResult = new HashMap<>(genericReadResults.size()); + for (SortableEntityData sd : genericReadResults) { + mergedResult.put(sd.getId(), sd); + } + for (GroupPermissions permissions : groupPermissions) { + SortableEntityData alreadyAdded = mergedResult.get(permissions.groupId); + if (alreadyAdded != null) { + alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs); + alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs); + } else { + EntityGroupData egData = repository.getEntityGroup(permissions.groupId); + if (matches(egData)) { + SortableEntityData sortData = toSortData(egData, permissions); + mergedResult.put(egData.getId(), sortData); + } + } + } + return new ArrayList<>(mergedResult.values()); + } + + @Override + protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List groupPermissions) { + for (GroupPermissions eg : groupPermissions) { + if (read && readAttrs && readTs) { + break; + } + if (eg.groupId.equals(id)) { + read = true; + readAttrs = readAttrs || eg.readAttrs; + readTs = readTs || eg.readTs; + } + } + return new CombinedPermissions(read, readAttrs, readTs); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java new file mode 100644 index 0000000000..e5e9dd6364 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileNameQueryProcessor.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileNames; + private final Pattern pattern; + + public AbstractEntityProfileNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + entityProfileNames = new HashSet<>(getProfileNames(this.filter)); + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && entityProfileNames.contains(ed.getFields().getType()) + && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java new file mode 100644 index 0000000000..96c3186358 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntityProfileQueryProcessor.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Pattern; + +public abstract class AbstractEntityProfileQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + private final Pattern pattern; + + public AbstractEntityProfileQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter, entityType); + var profileNamesSet = new HashSet<>(getProfileNames(this.filter)); + for (EntityData dp : repo.getEntitySet(getProfileEntityType())) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + pattern = RepositoryUtils.toSqlLikePattern(getEntityNameFilter(filter)); + } + + protected abstract String getEntityNameFilter(T filter); + + protected abstract List getProfileNames(T filter); + + protected abstract EntityType getProfileEntityType(); + + @Override + protected boolean matches(EntityData ed) { + ProfileAwareData profileAwareData = (ProfileAwareData) ed; + return super.matches(ed) && entityProfileIds.contains(profileAwareData.getFields().getProfileId()) + && (pattern == null || pattern.matcher(profileAwareData.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java new file mode 100644 index 0000000000..a5f7621d43 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractEntitySearchQueryProcessor.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitySearchQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; + +public abstract class AbstractEntitySearchQueryProcessor extends AbstractRelationQueryProcessor { + + + public AbstractEntitySearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + @Override + public Set getRootEntities() { + return Set.of(filter.getRootEntity().getId()); + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + public abstract EntityType getEntityType(); + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData target = relationInfo.getTarget(); + return (filter.getRelationType() == null || relationInfo.getType().equals(filter.getRelationType())) && + getEntityType().equals(target.getEntityType()) && super.matches(target); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java new file mode 100644 index 0000000000..9efc005df2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractQueryProcessor.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupTypePermissionInfo; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.checkFilters; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractQueryProcessor implements EntityQueryProcessor { + + protected final TenantRepo repository; + protected final QueryContext ctx; + protected final EdqsQuery query; + protected final DataKey sortKey; + protected final T filter; + + public AbstractQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + this.repository = repository; + this.ctx = ctx; + this.query = query; + this.sortKey = query instanceof EdqsDataQuery dataQuery ? dataQuery.getSortKey() : null; + this.filter = filter; + } + + protected CombinedPermissions getCombinedPermissions(UUID id, boolean genericRead, boolean genericAttrs, boolean genericTs, List groupPermissions) { + return getCombinedPermissionsInternal(id, genericRead, genericRead && genericAttrs, genericRead && genericTs, groupPermissions); + } + + protected CombinedPermissions getCombinedPermissions(UUID id, List groupPermissions) { + return getCombinedPermissionsInternal(id, false, false, false, groupPermissions); + } + + protected CombinedPermissions getCombinedPermissionsInternal(UUID id, boolean read, boolean readAttrs, boolean readTs, List groupPermissions) { + for (GroupPermissions eg : groupPermissions) { + if (read && readAttrs && readTs) { + break; + } + boolean hasMorePermissions = !read || (!readAttrs && eg.readAttrs) || (!readTs && eg.readTs); + if (hasMorePermissions && repository.contains(eg.groupId, id)) { + read = true; + readAttrs = readAttrs || eg.readAttrs; + readTs = readTs || eg.readTs; + } + } + return new CombinedPermissions(read, readAttrs, readTs); + } + + protected SortableEntityData toSortDataGroupsOnly(EntityData ed, List groupPermissions) { + SortableEntityData sortData; + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), groupPermissions); + if (permissions.isRead()) { + sortData = toSortData(ed, permissions); + } else { + sortData = null; + } + return sortData; + } + + protected SortableEntityData toSortData(EntityData ed, boolean readAttrs, boolean readTs) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(readAttrs); + sortData.setReadTs(readTs); + return sortData; + } + + protected SortableEntityData toSortData(EntityData ed, Permissions permissions) { + return toSortData(ed, permissions.isReadAttrs(), permissions.isReadTs()); + } + + protected static List toGroupPermissions(MergedGroupTypePermissionInfo readPermissions, + MergedGroupTypePermissionInfo readAttrPermissions, + MergedGroupTypePermissionInfo readTsPermissions) { + List permissions = new ArrayList<>(); + for (EntityGroupId egId : readPermissions.getEntityGroupIds()) { + permissions.add(new GroupPermissions(egId.getId(), + readAttrPermissions.getEntityGroupIds() != null && readAttrPermissions.getEntityGroupIds().contains(egId), + readTsPermissions.getEntityGroupIds() != null && readTsPermissions.getEntityGroupIds().contains(egId))); + } + return permissions; + } + + protected static boolean checkCustomerHierarchy(Set customers, EntityData ed) { + return ed.getCustomerId() != null && customers.contains(ed.getCustomerId()); + } + + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + processor.accept(ed); + } + } + } + + protected boolean matches(EntityData ed) { + return checkFilters(query, ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java new file mode 100644 index 0000000000..755c79cea6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractRelationQueryProcessor.java @@ -0,0 +1,260 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; +import java.util.UUID; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public abstract class AbstractRelationQueryProcessor extends AbstractQueryProcessor { + + public static final int MAXIMUM_QUERY_LEVEL = 100; + + public AbstractRelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter) { + super(repo, ctx, query, filter); + } + + protected abstract Set getRootEntities(); + + protected abstract EntitySearchDirection getDirection(); + + protected abstract int getMaxLevel(); + + protected abstract boolean isFetchLastLevelOnly(); + + protected boolean isMultiRoot() { + return false; + } + + @Override + public List processQuery() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + if (ctx.isTenantUser()) { + return processTenantQuery(entities); + } else { + return processCustomerQuery(entities); + } + } + + @Override + public long count() { + var relations = repository.getRelations(RelationTypeGroup.COMMON); + var entities = getEntitiesSet(relations); + long result = 0; + + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + if (ctx.isTenantUser()) { + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + result++; + } + } else if (permissions.isReadEntity()) { + result++; + } + } + } + } else { + var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId()); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId()); + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + isReadEntity, + permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + result++; + } + } else if (isReadEntity) { + result++; + } + } + } + return result; + } + return result; + } + + private List processTenantQuery(Set> entities) { + List result = new ArrayList<>(); + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + if (permissions.isHasGroups()) { + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + permissions.isReadEntity(), permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(combinedPermissions.isReadAttrs()); + sortData.setReadTs(combinedPermissions.isReadTs()); + result.add(sortData); + } + } else if (permissions.isReadEntity()) { + result.add(toSortData(ed, permissions)); + } + } + } + return result; + } + + private List processCustomerQuery(Set> entities) { + var customerIds = repository.getAllCustomers(ctx.getCustomerId().getId()); + RelationQueryPermissions[] permissionsArray = buildPermissionsArray(); + List result = new ArrayList<>(); + for (EntityData ed : entities) { + var permissions = permissionsArray[ed.getEntityType().ordinal()]; + if (permissions != null) { + boolean isReadEntity = permissions.isReadEntity() && ed.getCustomerId() != null && customerIds.contains(ed.getCustomerId()); + if (permissions.isHasGroups()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + CombinedPermissions combinedPermissions = getCombinedPermissions(ed.getId(), + isReadEntity, + permissions.isReadAttrs(), permissions.isReadTs(), permissions.getGroupPermissions()); + if (combinedPermissions.isRead()) { + sortData.setReadAttrs(combinedPermissions.isReadAttrs()); + sortData.setReadTs(combinedPermissions.isReadTs()); + result.add(sortData); + } + } else if (isReadEntity) { + result.add(toSortData(ed, permissions)); + } + } + } + return result; + } + + private RelationQueryPermissions[] buildPermissionsArray() { + RelationQueryPermissions[] permissionsArray = new RelationQueryPermissions[EntityType.values().length]; + var readEntityPermissionsMap = ctx.getMergedReadEntityPermissionsMap(); + var readAttrPermissionsMap = ctx.getMergedReadAttrPermissionsMap(); + var readTsPermissionsMap = ctx.getMergedReadTsPermissionsMap(); + for (EntityType et : EntityType.values()) { + var resource = Resource.resourceFromEntityType(et); + if (resource == null) { + continue; + } + var readEntityPermissions = readEntityPermissionsMap.get(resource); + var readAttrPermissions = readAttrPermissionsMap.get(resource); + var readTsPermissions = readTsPermissionsMap.get(resource); + var groupPermissions = toGroupPermissions(readEntityPermissions, readAttrPermissions, readTsPermissions); + RelationQueryPermissions entityPermissions = RelationQueryPermissions + .builder() + .readEntity(readEntityPermissions.isHasGenericRead()) + .readAttrs(readAttrPermissions.isHasGenericRead()) + .readTs(readTsPermissions.isHasGenericRead()) + .hasGroups(!groupPermissions.isEmpty()) + .groupPermissions(groupPermissions) + .build(); + permissionsArray[et.ordinal()] = entityPermissions; + } + return permissionsArray; + } + + private Set> getEntitiesSet(RelationsRepo relations) { + Set> result = new HashSet<>(); + Set processed = new HashSet<>(); + Queue tasks = new LinkedList<>(); + int maxLvl = getMaxLevel() == 0 ? MAXIMUM_QUERY_LEVEL : Math.max(1, getMaxLevel()); + for (UUID uuid : getRootEntities()) { + tasks.add(new RelationSearchTask(uuid, 0)); + } + while (!tasks.isEmpty()) { + RelationSearchTask task = tasks.poll(); + if (processed.add(task.entityId)) { + var entityLvl = task.lvl + 1; + Set entities = EntitySearchDirection.FROM.equals(getDirection()) ? relations.getFrom(task.entityId) : relations.getTo(task.entityId); + if (isFetchLastLevelOnly() && entities.isEmpty() && task.previous != null && check(task.previous)) { + result.add(task.previous.getTarget()); + } + for (RelationInfo relationInfo : entities) { + var entity = relationInfo.getTarget(); + if (entity.isEmpty()) { + continue; + } + var entityId = entity.getId(); + if (isFetchLastLevelOnly()) { + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl, relationInfo)); + } else if (entityLvl == maxLvl) { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + } + } else { + if (check(relationInfo)) { + if (isMultiRoot()) { + ctx.getRelatedParentIdMap().put(entity.getId(), task.entityId); + } + result.add(entity); + } + if (entityLvl < maxLvl) { + tasks.add(new RelationSearchTask(entityId, entityLvl)); + } + } + } + } + } + return result; + } + + protected abstract boolean check(RelationInfo relationInfo); + + @RequiredArgsConstructor + private static class RelationSearchTask { + private final UUID entityId; + private final int lvl; + private final RelationInfo previous; + + public RelationSearchTask(UUID entityId, int lvl) { + this(entityId, lvl, null); + } + + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java new file mode 100644 index 0000000000..2a9ef090e7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSimpleQueryProcessor.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; + +public abstract class AbstractSimpleQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + + public AbstractSimpleQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query, T filter, EntityType entityType) { + super(repo, ctx, query, filter); + this.entityType = entityType; + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(entityType), processor); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + var genericReadResults = processCustomerGenericRead(customerId, readAttrPermissions, readTsPermissions); + Map mergedResult = new HashMap<>(genericReadResults.size()); + for (SortableEntityData sd : genericReadResults) { + mergedResult.put(sd.getId(), sd); + } + + for (GroupPermissions permissions : groupPermissions) { + EntityGroupData egData = repository.getEntityGroup(permissions.groupId); + for (EntityData ed : egData.getEntities()) { + SortableEntityData alreadyAdded = mergedResult.get(ed.getId()); + if (alreadyAdded != null) { + alreadyAdded.setReadAttrs(alreadyAdded.isReadAttrs() || permissions.readAttrs); + alreadyAdded.setReadTs(alreadyAdded.isReadTs() || permissions.readTs); + } else { + if (matches(ed)) { + SortableEntityData sortData = toSortData(ed, permissions); + mergedResult.put(ed.getId(), sortData); + } + } + } + } + return new ArrayList<>(mergedResult.values()); + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + for (GroupPermissions groupPermission : groupPermissions) { + EntityGroupData egData = repository.getEntityGroup(groupPermission.groupId); + process(egData.getEntities(), processor); + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(entityType), processor); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java new file mode 100644 index 0000000000..9411d75935 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AbstractSingleEntityTypeQueryProcessor.java @@ -0,0 +1,172 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; + +public abstract class AbstractSingleEntityTypeQueryProcessor extends AbstractQueryProcessor { + + public AbstractSingleEntityTypeQueryProcessor(TenantRepo repository, QueryContext ctx, EdqsQuery query, T filter) { + super(repository, ctx, query, filter); + } + + @Override + public List processQuery() { + var readPermissions = ctx.getMergedReadPermissionsByEntityType(); + if (readPermissions == null) { + return Collections.emptyList(); + } + var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType(); + var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType(); + + boolean hasGenericRead = readPermissions.isHasGenericRead(); + boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty(); + if (!hasGenericRead && !hasGroups) { + return Collections.emptyList(); + } + boolean hasGenericAttrRead = readAttrPermissions.isHasGenericRead(); + boolean hasGenericTsRead = readTsPermissions.isHasGenericRead(); + + if (hasGenericRead) { + if (ctx.isTenantUser()) { + if (hasGroups && (!hasGenericAttrRead || !hasGenericTsRead)) { + return processTenantGenericReadWithGroups(hasGenericAttrRead, hasGenericTsRead, + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } else { + return processTenantGenericRead(hasGenericAttrRead, hasGenericTsRead); + } + } else { + if (hasGroups) { + return processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead, + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } else { + return processCustomerGenericRead(ctx.getCustomerId().getId(), hasGenericAttrRead, hasGenericTsRead); + } + } + } else { + return processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)); + } + } + + @Override + public long count() { // TODO: get rid of the duplicates + var readPermissions = ctx.getMergedReadPermissionsByEntityType(); + if (readPermissions == null) { + return 0; + } + var readAttrPermissions = ctx.getMergedReadAttrPermissionsByEntityType(); + var readTsPermissions = ctx.getMergedReadTsPermissionsByEntityType(); + boolean hasGenericRead = readPermissions.isHasGenericRead(); + boolean hasGroups = readPermissions.getEntityGroupIds() != null && !readPermissions.getEntityGroupIds().isEmpty(); + + if (!hasGenericRead && !hasGroups && !ctx.isIgnorePermissionCheck()) { + return 0; + } + + AtomicLong result = new AtomicLong(); + Consumer> counter = ed -> result.incrementAndGet(); + + if (ctx.isIgnorePermissionCheck()) { + processAll(counter); + } else if (ctx.isTenantUser()) { + if (hasGenericRead) { + processAll(counter); + } else { + processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter); + } + } else { + if (hasGenericRead) { + if (hasGroups) { + result.addAndGet(processCustomerGenericReadWithGroups(ctx.getCustomerId().getId(), readAttrPermissions.isHasGenericRead(), readTsPermissions.isHasGenericRead(), + toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions)).size()); // FIXME: not efficient + } else { + processCustomerGenericRead(ctx.getCustomerId().getId(), counter); + } + } else { + processGroupsOnly(toGroupPermissions(readPermissions, readAttrPermissions, readTsPermissions), counter); + } + } + return result.get(); + } + + protected List processTenantGenericRead(boolean readAttrPermissions, + boolean readTsPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + result.add(toSortData(ed, readAttrPermissions, readTsPermissions)); + }); + return result; + } + + protected List processCustomerGenericRead(UUID customerId, + boolean readAttrPermissions, + boolean readTsPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processCustomerGenericRead(customerId, ed -> { + result.add(toSortData(ed, readAttrPermissions, readTsPermissions)); + }); + return result; + } + + protected abstract void processCustomerGenericRead(UUID customerId, Consumer> processor); + + protected List processTenantGenericReadWithGroups(boolean readAttrPermissions, + boolean readTsPermissions, + List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), true, readAttrPermissions, readTsPermissions, groupPermissions); + SortableEntityData sortData = toSortData(ed, permissions); + result.add(sortData); + }); + return result; + } + + protected abstract List processCustomerGenericReadWithGroups(UUID customerId, + boolean readAttrPermissions, + boolean readTsPermissions, + List groupPermissions); + + protected List processGroupsOnly(List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + processGroupsOnly(groupPermissions, ed -> { + SortableEntityData sortData = toSortDataGroupsOnly(ed, groupPermissions); + if (sortData != null) { + result.add(sortData); + } + }); + return result; + } + + protected abstract void processGroupsOnly(List groupPermissions, Consumer> processor); + + protected abstract void processAll(Consumer> processor); + + protected abstract int getProbableResultSize(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java new file mode 100644 index 0000000000..c2821161c7 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + public ApiUsageStateQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (ApiUsageStateFilter) query.getEntityFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + process(customerData.getEntities(EntityType.API_USAGE_STATE), processor); + } + + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + CustomerData customerData = (CustomerData) repository.getEntityMap(EntityType.CUSTOMER).get(customerId); + Collection> entities = customerData.getEntities(EntityType.API_USAGE_STATE); + EntityData ed = entities.iterator().next(); + if (entities.isEmpty() || !matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(customerData, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(EntityType.API_USAGE_STATE), processor); + } + + @Override + protected boolean matches(EntityData ed) { + ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); + return super.matches(ed) && (filter.getCustomerId() != null ? entityFields.getEntityId().equals(filter.getCustomerId()) : + entityFields.getEntityId().equals(repository.getTenantId())); + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java new file mode 100644 index 0000000000..89826fe1be --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public AssetSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getAssetTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getAssetTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.ASSET_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java new file mode 100644 index 0000000000..d012e12e46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/AssetTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class AssetTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public AssetTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (AssetTypeFilter) query.getEntityFilter(), EntityType.ASSET); + } + + @Override + protected String getEntityNameFilter(AssetTypeFilter filter) { + return filter.getAssetNameFilter(); + } + + @Override + protected List getProfileNames(AssetTypeFilter filter) { + return filter.getAssetTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.ASSET_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java new file mode 100644 index 0000000000..3ff6879d46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/CombinedPermissions.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Data; + +@Data +public class CombinedPermissions implements Permissions { + private final boolean read; + private final boolean readAttrs; + private final boolean readTs; +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java new file mode 100644 index 0000000000..e76f0159bf --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceSearchQueryProcessor.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.ProfileAwareData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + private final Set entityProfileIds = new HashSet<>(); + + public DeviceSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceSearchQueryFilter) query.getEntityFilter()); + if (CollectionsUtil.isNotEmpty(filter.getDeviceTypes())) { + var profileNamesSet = new HashSet<>(this.filter.getDeviceTypes()); + for (EntityData dp : repo.getEntitySet(EntityType.DEVICE_PROFILE)) { + if (profileNamesSet.contains(dp.getFields().getName())) { + entityProfileIds.add(dp.getId()); + } + } + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + return super.check(relationInfo) && + (entityProfileIds.isEmpty() || entityProfileIds.contains(((ProfileAwareData) relationInfo.getTarget()).getFields().getProfileId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java new file mode 100644 index 0000000000..19068eb160 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/DeviceTypeQueryProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class DeviceTypeQueryProcessor extends AbstractEntityProfileQueryProcessor { + + public DeviceTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (DeviceTypeFilter) query.getEntityFilter(), EntityType.DEVICE); + } + + @Override + protected String getEntityNameFilter(DeviceTypeFilter filter) { + return filter.getDeviceNameFilter(); + } + + @Override + protected List getProfileNames(DeviceTypeFilter filter) { + return filter.getDeviceTypes(); + } + + @Override + protected EntityType getProfileEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java new file mode 100644 index 0000000000..2c5c7b1dd2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EdgeTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EdgeTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeTypeFilter) query.getEntityFilter(), EntityType.EDGE); + } + + @Override + protected String getEntityNameFilter(EdgeTypeFilter filter) { + return filter.getEdgeNameFilter(); + } + + @Override + protected List getProfileNames(EdgeTypeFilter filter) { + return filter.getEdgeTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java new file mode 100644 index 0000000000..3ea4c247a9 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EdgeTypeSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EdgeTypeSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EdgeTypeSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EdgeSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEdgeTypes() == null || filter.getEdgeTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java new file mode 100644 index 0000000000..cc3c26dc18 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupNameQueryProcessor.java @@ -0,0 +1,121 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP; +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntitiesByGroupNameQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final String groupType; + private final UUID ownerId; + private final Pattern pattern; + + public EntitiesByGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntitiesByGroupNameFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.ownerId = filter.getOwnerId() != null ? filter.getOwnerId().getId() : null; + this.pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(ENTITY_GROUP), processor); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + Collection> entities = new HashSet<>(getProbableResultSize()); + for (GroupPermissions groupPermission : groupPermissions) { + entities.add(repository.getEntityGroup(groupPermission.groupId)); + } + process(entities, processor); + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(ENTITY_GROUP), processor); + } + + @Override + protected void process(Collection> entities, Consumer> processor) { + for (EntityData ed : entities) { + if (matches(ed)) { + Collection> groupEntities = repository.getEntityGroup(ed.getId()).getEntities(); + for (EntityData groupEntity : groupEntities) { + processor.accept(groupEntity); + } + return; + } + } + } + + @Override + protected boolean matches(EntityData ed) { + EntityGroupFields fields = (EntityGroupFields)ed.getFields(); + return super.matches(ed) && groupType.equals(fields.getType()) + && (pattern == null || pattern.matcher(fields.getName()).matches()) + && (ownerId == null || ownerId.equals(fields.getOwnerId())); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java new file mode 100644 index 0000000000..feb51e5f46 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntitiesByGroupQueryProcessor.java @@ -0,0 +1,96 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntitiesByGroupQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final String groupType; + private final UUID groupId; + + public EntitiesByGroupQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupFilter) query.getEntityFilter()); + groupId = UUID.fromString(filter.getEntityGroup()); + groupType = filter.getGroupType().name(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityGroupData entityGroup = repository.getEntityGroup(groupId); + if (matches(entityGroup)) { + for (EntityData ed : entityGroup.getEntities()) { + processor.accept(ed); + } + } + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && groupType.equals(ed.getFields().getType()); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java new file mode 100644 index 0000000000..7022ca5cb6 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupListQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class EntityGroupListQueryProcessor extends AbstractEntityGroupQueryProcessor { + + private final String groupType; + private final Set groupIds; + + public EntityGroupListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupListFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.groupIds = filter.getEntityGroupList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + Set allowedGroupIds = groupPermissions.stream().map(GroupPermissions::getGroupId) + .filter(this.groupIds::contains).collect(Collectors.toSet()); + + checkGroupIds(allowedGroupIds, processor); + } + + @Override + protected void processAll(Consumer> processor) { + checkGroupIds(groupIds, processor); + } + + @Override + protected int getProbableResultSize() { + return groupIds.size(); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && groupType.equals(ed.getFields().getType()); + } + + private void checkGroupIds(Set allowedGroupIds, Consumer> processor) { + for (UUID groupId : allowedGroupIds) { + EntityGroupData entityGroup = repository.getEntityGroup(groupId); + if (matches(entityGroup)) { + processor.accept(entityGroup); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java new file mode 100644 index 0000000000..9491142619 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityGroupNameQueryProcessor.java @@ -0,0 +1,83 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityGroupNameFilter; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.regex.Pattern; + +import static org.thingsboard.server.common.data.EntityType.ENTITY_GROUP; + +public class EntityGroupNameQueryProcessor extends AbstractEntityGroupQueryProcessor { + + private final String groupType; + private final Pattern groupNamePattern; + + public EntityGroupNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityGroupNameFilter) query.getEntityFilter()); + this.groupType = filter.getGroupType().name(); + this.groupNamePattern = RepositoryUtils.toSqlLikePattern(filter.getEntityGroupNameFilter()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getEntityMap(EntityType.CUSTOMER); + for (UUID cId : repository.getAllCustomers(customerId)) { + var customerData = (CustomerData) customers.get(cId); + if (customerData != null) { + process(customerData.getEntities(ENTITY_GROUP), processor); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + for (GroupPermissions groupPermission : groupPermissions) { + EntityGroupData entityGroup = repository.getEntityGroup(groupPermission.groupId); + if (matches(entityGroup)) { + processor.accept(entityGroup); + } + } + } + + @Override + protected void processAll(Consumer> processor) { + process(repository.getEntitySet(ENTITY_GROUP), processor); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && (groupNamePattern == null || groupNamePattern.matcher(ed.getFields().getName()).matches()) + && groupType.equals(ed.getFields().getType()); + } + + @Override + protected int getProbableResultSize() { + return 1024; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java new file mode 100644 index 0000000000..495f925bd0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityListQueryProcessor.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.getSortValue; + +public class EntityListQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final Set entityIds; + + public EntityListQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityListFilter) query.getEntityFilter()); + this.entityType = filter.getEntityType(); + this.entityIds = filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toSet()); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + if (checkCustomerHierarchy(customers, ed)) { + processor.accept(ed); + } + }); + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + List result = new ArrayList<>(getProbableResultSize()); + var customers = repository.getAllCustomers(customerId); + processAll(ed -> { + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), checkCustomerHierarchy(customers, ed), readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = new SortableEntityData(ed); + sortData.setSortValue(getSortValue(ed, sortKey)); + sortData.setReadAttrs(permissions.isReadAttrs()); + sortData.setReadTs(permissions.isReadTs()); + result.add(sortData); + } + }); + return result; + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + var map = repository.getEntityMap(entityType); + for (UUID entityId : entityIds) { + EntityData ed = map.get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + } + + @Override + protected int getProbableResultSize() { + return entityIds.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java new file mode 100644 index 0000000000..3b64706bfe --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityNameQueryProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.regex.Pattern; + +public class EntityNameQueryProcessor extends AbstractSimpleQueryProcessor { + + private final Pattern pattern; + + public EntityNameQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityNameFilter) query.getEntityFilter(), ((EntityNameFilter) query.getEntityFilter()).getEntityType()); + pattern = RepositoryUtils.toSqlLikePattern(filter.getEntityNameFilter()); + } + + @Override + protected boolean matches(EntityData ed) { + return ed.getFields() != null && (pattern == null || pattern.matcher(ed.getFields().getName()).matches()); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java new file mode 100644 index 0000000000..a42c79b61f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.edqs.query.SortableEntityData; + +import java.util.List; + +public interface EntityQueryProcessor { + + List processQuery(); + + long count(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java new file mode 100644 index 0000000000..271ff01425 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityQueryProcessorFactory.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityQueryProcessorFactory { + + public static EntityQueryProcessor create(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + return switch (query.getEntityFilter().getType()) { + case SINGLE_ENTITY -> new SingleEntityQueryProcessor(repo, ctx, query); + case ENTITY_LIST -> new EntityListQueryProcessor(repo, ctx, query); + case ENTITY_NAME -> new EntityNameQueryProcessor(repo, ctx, query); + case ENTITY_TYPE -> new EntityTypeQueryProcessor(repo, ctx, query); + case DEVICE_TYPE -> new DeviceTypeQueryProcessor(repo, ctx, query); + case ASSET_TYPE -> new AssetTypeQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_TYPE -> new EntityViewTypeQueryProcessor(repo, ctx, query); + case EDGE_TYPE -> new EdgeTypeQueryProcessor(repo, ctx, query); + case RELATIONS_QUERY -> new RelationQueryProcessor(repo, ctx, query); + case ENTITY_GROUP -> new EntitiesByGroupQueryProcessor(repo, ctx, query); + case ENTITY_GROUP_LIST -> new EntityGroupListQueryProcessor(repo, ctx, query); + case ENTITY_GROUP_NAME -> new EntityGroupNameQueryProcessor(repo, ctx, query); + case ENTITIES_BY_GROUP_NAME -> new EntitiesByGroupNameQueryProcessor(repo, ctx, query); + case STATE_ENTITY_OWNER -> new StateEntityOwnerQueryProcessor(repo, ctx, query); + case API_USAGE_STATE -> new ApiUsageStateQueryProcessor(repo, ctx, query); + case ASSET_SEARCH_QUERY -> new AssetSearchQueryProcessor(repo, ctx, query); + case DEVICE_SEARCH_QUERY -> new DeviceSearchQueryProcessor(repo, ctx, query); + case ENTITY_VIEW_SEARCH_QUERY -> new EntityViewSearchQueryProcessor(repo, ctx, query); + case EDGE_SEARCH_QUERY -> new EdgeTypeSearchQueryProcessor(repo, ctx, query); + case SCHEDULER_EVENT -> new SchedulerEventQueryProcessor(repo, ctx, query); + default -> throw new RuntimeException("Not Implemented!"); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java new file mode 100644 index 0000000000..b390a3c80c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityTypeQueryProcessor.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityTypeQueryProcessor extends AbstractSimpleQueryProcessor { + + public EntityTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityTypeFilter) query.getEntityFilter(), ((EntityTypeFilter) query.getEntityFilter()).getEntityType()); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java new file mode 100644 index 0000000000..f16de48b6a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewSearchQueryProcessor.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class EntityViewSearchQueryProcessor extends AbstractEntitySearchQueryProcessor { + + public EntityViewSearchQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewSearchQueryFilter) query.getEntityFilter()); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + + @Override + protected boolean check(RelationInfo relationInfo) { + EntityData ed = relationInfo.getTarget(); + return super.check(relationInfo) && + (filter.getEntityViewTypes() == null || filter.getEntityViewTypes().contains(ed.getFields().getType())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java new file mode 100644 index 0000000000..3932cd15fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/EntityViewTypeQueryProcessor.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.List; + +public class EntityViewTypeQueryProcessor extends AbstractEntityProfileNameQueryProcessor { + + public EntityViewTypeQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (EntityViewTypeFilter) query.getEntityFilter(), EntityType.ENTITY_VIEW); + } + + @Override + protected String getEntityNameFilter(EntityViewTypeFilter filter) { + return filter.getEntityViewNameFilter(); + } + + @Override + protected List getProfileNames(EntityViewTypeFilter filter) { + return filter.getEntityViewTypes(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java new file mode 100644 index 0000000000..65a5a7d9b0 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/GroupPermissions.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Data; + +import java.util.UUID; + +@Data +public class GroupPermissions implements Permissions { + + protected final UUID groupId; + protected final boolean readAttrs; + protected final boolean readTs; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java new file mode 100644 index 0000000000..e2231a8df2 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/Permissions.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +public interface Permissions { + + boolean isReadAttrs(); + + boolean isReadTs(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java new file mode 100644 index 0000000000..fb151d85af --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryPermissions.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import lombok.Builder; +import lombok.Getter; + +import java.util.List; + +@Getter +@Builder +public class RelationQueryPermissions implements Permissions { + + private final boolean readEntity; + private final boolean readAttrs; + private final boolean readTs; + private final boolean hasGroups; + private final List groupPermissions; + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java new file mode 100644 index 0000000000..8f70344239 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/RelationQueryProcessor.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.edqs.data.RelationInfo; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +public class RelationQueryProcessor extends AbstractRelationQueryProcessor { + + private final boolean hasFilters; + + public RelationQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (RelationsQueryFilter) query.getEntityFilter()); + this.hasFilters = filter.getFilters() != null && !filter.getFilters().isEmpty(); + } + + @Override + public Set getRootEntities() { + if (filter.isMultiRoot()) { + return filter.getMultiRootEntityIds().stream().map(UUID::fromString).collect(Collectors.toSet()); + } else { + return Set.of(filter.getRootEntity().getId()); + } + } + + @Override + public EntitySearchDirection getDirection() { + return filter.getDirection(); + } + + @Override + public int getMaxLevel() { + return filter.getMaxLevel(); + } + + @Override + public boolean isMultiRoot() { + return filter.isMultiRoot(); + } + + @Override + public boolean isFetchLastLevelOnly() { + return filter.isFetchLastLevelOnly(); + } + + @Override + protected boolean check(RelationInfo relationInfo) { + if (hasFilters) { + for (var f : filter.getFilters()) { + if (((!filter.isNegate() && !f.isNegate()) || (filter.isNegate() && f.isNegate())) == f.getRelationType().equals(relationInfo.getType())) { + if (f.getEntityTypes() == null || f.getEntityTypes().isEmpty() + || f.getEntityTypes().contains(relationInfo.getTarget().getEntityType())) { + return super.matches(relationInfo.getTarget()); + } + } + } + return false; + } else { + return super.matches(relationInfo.getTarget()); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java new file mode 100644 index 0000000000..3c48b89cb5 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SchedulerEventQueryProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SchedulerEventFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.repo.TenantRepo; + +public class SchedulerEventQueryProcessor extends AbstractSimpleQueryProcessor { + + public SchedulerEventQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SchedulerEventFilter) query.getEntityFilter(), EntityType.SCHEDULER_EVENT); + } + + @Override + protected boolean matches(EntityData ed) { + return super.matches(ed) && (filter.getEventType() == null || filter.getEventType().equals(ed.getFields().getType())) + && (filter.getOriginator() == null || filter.getOriginator().equals(ed.getFields().getOriginatorId())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java new file mode 100644 index 0000000000..ee3810ca6b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/SingleEntityQueryProcessor.java @@ -0,0 +1,87 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class SingleEntityQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityType entityType; + private final UUID entityId; + + public SingleEntityQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (SingleEntityFilter) query.getEntityFilter()); + this.entityType = filter.getSingleEntity().getEntityType(); + this.entityId = filter.getSingleEntity().getId(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (ed != null && ed.getCustomerId() != null && matches(ed)) { + if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) { + processor.accept(ed); + } + } + } + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (!matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(ed, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityType).get(entityId); + if (matches(ed)) { + processor.accept(ed); + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java new file mode 100644 index 0000000000..0e79f21747 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/StateEntityOwnerQueryProcessor.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.query.processor; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.TenantRepo; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; + +public class StateEntityOwnerQueryProcessor extends AbstractSingleEntityTypeQueryProcessor { + + private final EntityId entityId; + + public StateEntityOwnerQueryProcessor(TenantRepo repo, QueryContext ctx, EdqsQuery query) { + super(repo, ctx, query, (StateEntityOwnerFilter) query.getEntityFilter()); + this.entityId = filter.getSingleEntity(); + } + + @Override + protected void processCustomerGenericRead(UUID customerId, Consumer> processor) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (ed != null && ed.getCustomerId() != null && matches(ed)) { + if (customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId())) { + processor.accept(ed); + } + } + } + + + @Override + protected List processCustomerGenericReadWithGroups(UUID customerId, boolean readAttrPermissions, boolean readTsPermissions, List groupPermissions) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (!matches(ed)) { + return Collections.emptyList(); + } else { + boolean genericRead = customerId.equals(ed.getCustomerId()) || repository.getAllCustomers(customerId).contains(ed.getCustomerId()); + CombinedPermissions permissions = getCombinedPermissions(ed.getId(), genericRead, readAttrPermissions, readTsPermissions, groupPermissions); + if (permissions.isRead()) { + SortableEntityData sortData = toSortData(ed, permissions); + return Collections.singletonList(sortData); + } else { + return Collections.emptyList(); + } + } + } + + @Override + protected void processGroupsOnly(List groupPermissions, Consumer> processor) { + processAll(processor); + } + + @Override + protected void processAll(Consumer> processor) { + EntityData ed = repository.getEntityMap(entityId.getEntityType()).get(entityId.getId()); + if (ed != null) { + if (ed.getCustomerId() != null) { + processor.accept(repository.getEntityMap(EntityType.CUSTOMER).get(ed.getCustomerId())); + } else { + processor.accept(repository.getEntityMap(EntityType.TENANT).get(repository.getTenantId().getId())); + } + } + } + + @Override + protected int getProbableResultSize() { + return 1; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java new file mode 100644 index 0000000000..9e829dfac1 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/EdqRepository.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +import java.util.function.Predicate; + +public interface EdqRepository { + + void processEvent(EdqsEvent event); + + @Deprecated + default void addOrUpdate(TenantId tenantId, Object object) {} + + long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck); + + PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck); + + void clearIf(Predicate predicate); + + void clear(); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java new file mode 100644 index 0000000000..2934fdbfba --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Predicate; + +@EdqsComponent +@AllArgsConstructor +@Service +@Slf4j +public class InMemoryEdqRepository implements EdqRepository { + + private final static ConcurrentMap repos = new ConcurrentHashMap<>(); + private final Optional statsService; + + public TenantRepo get(TenantId tenantId) { + return repos.computeIfAbsent(tenantId, id -> new TenantRepo(id, statsService)); + } + + @Override + public void processEvent(EdqsEvent event) { + get(event.getTenantId()).processEvent(event); + } + + @Override + public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + long result = 0; + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + for (TenantRepo repo : repos.values()) { + result += repo.countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck); + } + } else { + result = get(tenantId).countEntitiesByQuery(customerId, userPermissions, query, ignorePermissionCheck); + } + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("countEntitiesByQuery: {} ms", timingMs); + return result; + } + + @Override + public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, + MergedUserPermissions userPermissions, EntityDataQuery query, boolean ignorePermissionCheck) { + long startNs = System.nanoTime(); + var result = get(tenantId).findEntityDataByQuery(customerId, userPermissions, query, ignorePermissionCheck); + double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; + log.info("findEntityDataByQuery: {} ms", timingMs); + return result; + } + + @Override + public void clearIf(Predicate predicate) { + repos.keySet().removeIf(predicate); + } + + @Override + public void clear() { + repos.clear(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java new file mode 100644 index 0000000000..f2dcd41e7c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/KeyDictionary.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + +public class KeyDictionary { + + private static final ConcurrentMap keyToIdDict = new ConcurrentHashMap<>(); + private static final ConcurrentMap idToKeyDict = new ConcurrentHashMap<>(); + private static final AtomicInteger keySeq = new AtomicInteger(); + + public static Integer get(String key) { + return keyToIdDict.computeIfAbsent(key, __ -> { + int keyId = keySeq.incrementAndGet(); + idToKeyDict.put(keyId, key); + return keyId; + }); + } + + public static String get(Integer keyId) { + return idToKeyDict.get(keyId); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java new file mode 100644 index 0000000000..876d92ee82 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbBytePool.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import com.google.common.hash.Hashing; +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbBytePool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static byte[] intern(byte[] data) { + if (data == null) { + return null; + } + var checksum = Hashing.sha512().hashBytes(data).toString(); + return pool.computeIfAbsent(checksum, c -> data); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java new file mode 100644 index 0000000000..177651248b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TbStringPool.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.util.concurrent.ConcurrentMap; + +public class TbStringPool { + + private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); + + public static String intern(String data) { + if (data == null) { + return null; + } + return pool.computeIfAbsent(data, str -> str); + } + + public static int size(){ + return pool.size(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java new file mode 100644 index 0000000000..93422eb5fd --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -0,0 +1,584 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.AttributeKv; +import org.thingsboard.server.common.data.edqs.EdqsEvent; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.Entity; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; +import org.thingsboard.server.common.data.edqs.fields.EntityGroupFields; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.data.ApiUsageStateData; +import org.thingsboard.server.edqs.data.AssetData; +import org.thingsboard.server.edqs.data.CustomerData; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.EntityGroupData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.GenericData; +import org.thingsboard.server.edqs.data.RelationsRepo; +import org.thingsboard.server.edqs.data.TenantData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; +import org.thingsboard.server.edqs.data.dp.CompressedStringDataPoint; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.JsonDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; +import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; +import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Queue; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_ASC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SORT_DESC; +import static org.thingsboard.server.edqs.util.RepositoryUtils.SYS_ADMIN_PERMISSIONS; +import static org.thingsboard.server.edqs.util.RepositoryUtils.resolveEntityType; + +@Slf4j +public class TenantRepo { + + public static final Comparator> CREATED_TIME_COMPARATOR = Comparator.comparingLong(o -> o.getFields() != null ? o.getFields().getCreatedTime() : 0); // FIXME: fields may be null at first + public static final Comparator> CREATED_TIME_AND_ID_COMPARATOR = CREATED_TIME_COMPARATOR + .thenComparing(EntityData::getId); + public static final Comparator> CREATED_TIME_AND_ID_DESC_COMPARATOR = CREATED_TIME_AND_ID_COMPARATOR.reversed(); + + private final ConcurrentMap>> entitySetByType = new ConcurrentHashMap<>(); + private final ConcurrentMap>> entityMapByType = new ConcurrentHashMap<>(); + private final ConcurrentMap> customersHierarchy = new ConcurrentHashMap<>(); + private final ConcurrentMap entityGroups = new ConcurrentHashMap<>(); + private final ConcurrentMap relations = new ConcurrentHashMap<>(); + + private final Lock entityUpdateLock = new ReentrantLock(); + + private final TenantId tenantId; + private final Optional edqsStatsService; + + public TenantRepo(TenantId tenantId, Optional edqsStatsService) { + this.tenantId = tenantId; + this.edqsStatsService = edqsStatsService; + } + + public void processEvent(EdqsEvent event) { + EdqsObject edqsObject = event.getObject(); + log.debug("[{}] Processing event: {}", tenantId, event); + if (event.getEventType() == EdqsEventType.UPDATED) { + addOrUpdate(edqsObject); + } else if (event.getEventType() == EdqsEventType.DELETED) { + remove(edqsObject); + } + } + + public void addOrUpdate(EdqsObject object) { + if (object instanceof EntityRelation relation) { + addOrUpdateRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + addOrUpdateAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + addOrUpdateLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + addOrUpdateEntity(entity); + } + } + + public void remove(EdqsObject object) { + if (object instanceof EntityRelation relation) { + removeRelation(relation); + } else if (object instanceof AttributeKv attributeKv) { + removeAttribute(attributeKv); + } else if (object instanceof LatestTsKv latestTsKv) { + removeLatestKv(latestTsKv); + } else if (object instanceof Entity entity) { + removeEntity(entity); + } + } + + private void addOrUpdateRelation(EntityRelation entity) { + entityUpdateLock.lock(); + try { + if (RelationTypeGroup.COMMON.equals(entity.getTypeGroup())) { + RelationsRepo repo = relations.computeIfAbsent(entity.getTypeGroup(), tg -> new RelationsRepo()); + EntityData from = getOrCreate(entity.getFrom()); + EntityData to = getOrCreate(entity.getTo()); + boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + } + } else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entity.getTypeGroup())) { + var eg = getEntityGroup(entity.getFrom().getId()); + if (eg != null) { + eg.addOrUpdate(getOrCreate(entity.getTo())); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + private void removeRelation(EntityRelation entityRelation) { + if (RelationTypeGroup.COMMON.equals(entityRelation.getTypeGroup())) { + RelationsRepo relationsRepo = relations.get(entityRelation.getTypeGroup()); + if (relationsRepo != null) { + boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + } + } + } else if (RelationTypeGroup.FROM_ENTITY_GROUP.equals(entityRelation.getTypeGroup())) { + var eg = getEntityGroup(entityRelation.getFrom().getId()); + if (eg != null) { + eg.remove(entityRelation.getTo().getId()); + } + } + } + + private void addOrUpdateEntity(Entity entity) { + entityUpdateLock.lock(); + try { + log.trace("[{}] addOrUpdateEntity: {}", tenantId, entity); + EntityFields fields = entity.getFields(); + UUID entityId = fields.getId(); + EntityType entityType = entity.getType(); + + EntityData entityData = getOrCreate(entityType, entityId); + processFields(fields); + entityData.setFields(entity.getFields()); + + switch (entity.getType()) { + case ENTITY_GROUP -> { + EntityGroupFields entityGroupFields = (EntityGroupFields) fields; + UUID ownerId = entityGroupFields.getOwnerId(); + if (EntityType.CUSTOMER.equals(entityGroupFields.getOwnerType())) { + entityData.setCustomerId(ownerId); + ((CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(ownerId, CustomerData::new)).addOrUpdate(entityData); + } + entityGroups.put(fields.getId(), (EntityGroupData) entityData); + } + case CUSTOMER -> { + CustomerFields customerFields = (CustomerFields) fields; + UUID newParentId = customerFields.getCustomerId(); // for customer, customerId is parentCustomerId + UUID oldParentId = entityData.getCustomerId(); + entityData.setCustomerId(newParentId); + if (entityIdMismatch(oldParentId, newParentId)) { + if (oldParentId != null) { + customersHierarchy.computeIfAbsent(oldParentId, id -> new HashSet<>()).remove(entityData.getId()); + } + if (newParentId != null) { + customersHierarchy.computeIfAbsent(newParentId, id -> new HashSet<>()).add(entityData.getId()); + } + } + } + default -> { + UUID newCustomerId = fields.getCustomerId(); + UUID oldCustomerId = entityData.getCustomerId(); + entityData.setCustomerId(newCustomerId); + if (entityIdMismatch(oldCustomerId, newCustomerId)) { + if (oldCustomerId != null) { + CustomerData old = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(oldCustomerId); + if (old != null) { + old.remove(entityData); + } + } + if (newCustomerId != null) { + CustomerData newData = (CustomerData) getEntityMap(EntityType.CUSTOMER).computeIfAbsent(newCustomerId, CustomerData::new); + newData.addOrUpdate(entityData); + } + } + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void removeEntity(Entity entity) { + entityUpdateLock.lock(); + try { + UUID entityId = entity.getFields().getId(); + EntityType entityType = entity.getType(); + EntityData removed = getEntityMap(entityType).remove(entityId); + if (removed != null) { + getEntitySet(entityType).remove(removed); + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + } + switch (entityType) { + case ENTITY_GROUP -> { + entityGroups.remove(entityId); + } + case CUSTOMER -> { + customersHierarchy.remove(entityId); + } + } + } finally { + entityUpdateLock.unlock(); + } + } + + public void addOrUpdateAttribute(AttributeKv attributeKv) { + var entityData = getOrCreate(attributeKv.getEntityId()); + if (entityData != null) { + KvEntry value = attributeKv.getValue(); + Integer keyId = KeyDictionary.get(attributeKv.getKey()); + boolean added = entityData.putAttr(keyId, attributeKv.getScope(), toDataPoint(attributeKv.getLastUpdateTs(), value)); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeAttribute(AttributeKv attributeKv) { + var entityData = get(attributeKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + } + } + } + + public void addOrUpdateLatestKv(LatestTsKv latestTsKv) { + var entityData = getOrCreate(latestTsKv.getEntityId()); + if (entityData != null) { + KvEntry value = latestTsKv.getValue(); + Integer keyId = KeyDictionary.get(latestTsKv.getKey()); + boolean added = entityData.putTs(keyId, toDataPoint(latestTsKv.getTs(), value)); + if (added) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + } + } + } + + private void removeLatestKv(LatestTsKv latestTsKv) { + var entityData = get(latestTsKv.getEntityId()); + if (entityData != null) { + boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); + if (removed) { + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + } + } + } + + private DataPoint toDataPoint(long ts, KvEntry kvEntry) { + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BoolDataPoint(ts, kvEntry.getBooleanValue().get()); + case STRING -> getStrDataPoint(ts, kvEntry.getStrValue().get()); + case LONG -> new LongDataPoint(ts, kvEntry.getLongValue().get()); + case DOUBLE -> new DoubleDataPoint(ts, kvEntry.getDoubleValue().get()); + case JSON -> getJsonDataPoint(ts, kvEntry.getJsonValue().get()); + }; + } + + public void processFields(EntityFields fields) { + if (fields instanceof AssetFields assetFields) { + assetFields.setType(TbStringPool.intern(assetFields.getType())); + } + } + + private static DataPoint getStrDataPoint(long ts, String strV) { + DataPoint dp; + if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + dp = new StringDataPoint(ts, strV); + } else { + dp = new CompressedStringDataPoint(ts, strV); + } + return dp; + } + + private static DataPoint getJsonDataPoint(long ts, String strV) { + DataPoint dp; + if (strV.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + dp = new JsonDataPoint(ts, strV); + } else { + dp = new CompressedJsonDataPoint(ts, strV); + } + return dp; + } + + public ConcurrentMap> getEntityMap(EntityType entityType) { + return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); + } + + //TODO: automatically remove entities that has nothing except the ID. + private EntityData getOrCreate(EntityId entityId) { + return getOrCreate(entityId.getEntityType(), entityId.getId()); + } + + private EntityData getOrCreate(EntityType entityType, UUID entityId) { + return getEntityMap(entityType).computeIfAbsent(entityId, id -> { + log.debug("[{}] Adding {} {}", tenantId, entityType, id); + EntityData entityData = constructEntityData(entityType, entityId); + getEntitySet(entityType).add(entityData); + edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + return entityData; + }); + } + + private EntityData get(EntityId entityId) { + return getEntityMap(entityId.getEntityType()).get(entityId.getId()); + } + + private EntityData constructEntityData(EntityType entityType, UUID id) { + EntityData entityData = switch (entityType) { + case DEVICE -> new DeviceData(id); + case ASSET -> new AssetData(id); + case DEVICE_PROFILE, ASSET_PROFILE -> new EntityProfileData(id, entityType); + case CUSTOMER -> new CustomerData(id); + case TENANT -> new TenantData(id); + case ENTITY_GROUP -> new EntityGroupData(id); + case API_USAGE_STATE -> new ApiUsageStateData(id); + default -> new GenericData(entityType, id); + }; + entityData.setRepo(this); + return entityData; + } + + private static boolean entityIdMismatch(UUID oldOrNull, UUID newOrNull) { + if (oldOrNull == null) { + return newOrNull != null; + } else { + return !oldOrNull.equals(newOrNull); + } + } + + public Set> getEntitySet(EntityType entityType) { + return entitySetByType.computeIfAbsent(entityType, et -> new ConcurrentSkipListSet<>(CREATED_TIME_AND_ID_DESC_COMPARATOR)); + } + + public PageData findEntityDataByQuery(CustomerId customerId, MergedUserPermissions userPermissions, + EntityDataQuery oldQuery, boolean ignorePermissionCheck) { + EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck); + if (ctx == null) { + return PageData.emptyPageData(); + } + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return sortAndConvert(query, queryProcessor.processQuery(), ctx); + } + + public long countEntitiesByQuery(CustomerId customerId, MergedUserPermissions userPermissions, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { + EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); + log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); + QueryContext ctx = buildContext(customerId, userPermissions, query.getEntityFilter(), ignorePermissionCheck); + if (ctx == null) { + return 0; + } + EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); + return queryProcessor.count(); + } + + private PageData sortAndConvert(EdqsDataQuery query, List data, QueryContext ctx) { + int totalSize = data.size(); + int totalPages = (int) Math.ceil((float) totalSize / query.getPageSize()); + int offset = query.getPage() * query.getPageSize(); + if (offset > totalSize) { + return new PageData<>(Collections.emptyList(), totalPages, totalSize, false); + } else { + Comparator comparator = EntityDataSortOrder.Direction.ASC.equals(query.getSortDirection()) ? SORT_ASC : SORT_DESC; + long startTs = System.nanoTime(); +// IMPLEMENTATION THAT IS BASED ON PRIORITY_QUEUE +// var requiredSize = Math.min(offset + query.getPageSize(), totalSize); +// PriorityQueue topN = new PriorityQueue<>(requiredSize, comparator.reversed()); +// for (SortableEntityData item : data) { +// topN.add(item); +// if (topN.size() > requiredSize) { +// topN.poll(); +// } +// } +// List result = new ArrayList<>(topN); +// Collections.reverse(result); +// result = result.subList(offset, requiredSize); +// IMPLEMENTATION THAT IS BASED ON TREE SET (For offset + query.getPageSize() << totalSize) + var requiredSize = Math.min(offset + query.getPageSize(), totalSize); + TreeSet topNSet = new TreeSet<>(comparator); + for (SortableEntityData sp : data) { + topNSet.add(sp); + if (topNSet.size() > requiredSize) { + topNSet.pollLast(); + } + } + var result = topNSet.stream().skip(offset).limit(query.getPageSize()).collect(Collectors.toList()); +// IMPLEMENTATION THAT IS BASED ON TIM SORT (For offset + query.getPageSize() > totalSize / 2) +// data.sort(comparator); +// var result = data.subList(offset, endIndex); + log.debug("EDQ Sorted in {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTs)); + return new PageData<>(toQueryResult(result, query, ctx), totalPages, totalSize, totalSize > requiredSize); + } + } + + private List toQueryResult(List data, EdqsDataQuery query, QueryContext ctx) { + long ts = System.currentTimeMillis(); + List results = new ArrayList<>(data.size()); + for (SortableEntityData entityData : data) { + Map> latest = new HashMap<>(); + for (var key : query.getEntityFields()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(EntityKeyType.ENTITY_FIELD, t -> new HashMap<>()).put(key.key(), v); + } + for (var key : query.getLatestValues()) { + DataPoint dp = entityData.getEntityData().getDataPoint(key, ctx); + TsValue v = RepositoryUtils.toTsValue(ts, dp); + latest.computeIfAbsent(key.type(), t -> new HashMap<>()).put(KeyDictionary.get(key.keyId()), v); + } + + results.add(new QueryResult(entityData.getEntityId(), entityData.isReadAttrs(), entityData.isReadTs(), latest)); + } + return results; + } + + private QueryContext buildContext(CustomerId customerId, MergedUserPermissions userPermissions, EntityFilter filter, boolean ignorePermissionCheck) { + QueryContext queryContext; + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), SYS_ADMIN_PERMISSIONS, filter, ignorePermissionCheck); + } else { + switch (filter.getType()) { + case STATE_ENTITY_OWNER: + var singleEntity = ((StateEntityOwnerFilter) filter).getSingleEntity(); + EntityData ed = getEntityMap(singleEntity.getEntityType()).get(singleEntity.getId()); + if (ed != null) { + EntityId owner = ed.getCustomerId() != null ? new CustomerId(ed.getCustomerId()) : tenantId; + queryContext = new QueryContext(tenantId, customerId, owner.getEntityType(), userPermissions, filter, owner, ignorePermissionCheck); + } else { + return null; + } + break; + case SINGLE_ENTITY: + SingleEntityFilter seFilter = (SingleEntityFilter) filter; + EntityId entityId = seFilter.getSingleEntity(); + if (entityId != null && entityId.getEntityType().equals(EntityType.ENTITY_GROUP)) { + EntityGroupData entityGroupData = entityGroups.get(entityId.getId()); + if (entityGroupData != null) { + queryContext = new QueryContext(tenantId, customerId, EntityType.ENTITY_GROUP, userPermissions, filter, entityGroupData.getEntityType(), ignorePermissionCheck); + } else { + return null; + } + } else { + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck); + } + break; + default: + queryContext = new QueryContext(tenantId, customerId, resolveEntityType(filter), userPermissions, filter, ignorePermissionCheck); + } + } + return queryContext; + } + + public TenantId getTenantId() { + return tenantId; + } + + public Set getAllCustomers(UUID customerId) { + Set result = new HashSet<>(); + Queue queue = new LinkedList<>(); + + if (customerId != null) { + queue.add(customerId); + } + + while (!queue.isEmpty()) { + UUID current = queue.poll(); + if (!result.contains(current)) { + result.add(current); + Set children = customersHierarchy.get(current); + if (children != null) { + queue.addAll(children); + } + } + } + + return result; + } + + public boolean contains(UUID entityGroupID, UUID entityId) { + var groupData = entityGroups.get(entityGroupID); + return groupData != null && groupData.getEntity(entityId) != null; + } + + public EntityGroupData getEntityGroup(UUID groupId) { + return entityGroups.get(groupId); + } + + public RelationsRepo getRelations(RelationTypeGroup relationTypeGroup) { + return relations.get(relationTypeGroup); + } + + public String getOwnerName(EntityId ownerId) { + if (ownerId == null || (EntityType.CUSTOMER.equals(ownerId.getEntityType()) && CustomerId.NULL_UUID.equals(ownerId.getId()))) { + ownerId = tenantId; + } + return getEntityName(ownerId); + } + + private String getEntityName(EntityId entityId) { + EntityType entityType = entityId.getEntityType(); + return switch (entityType) { + case CUSTOMER, TENANT -> getEntityMap(entityType).get(entityId.getId()).getFields().getName(); + default -> throw new RuntimeException("Unsupported entity type: " + entityType); + }; + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java new file mode 100644 index 0000000000..e36e5ae5ab --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; + +import java.util.Set; + +public interface EdqsStateService { + + void restore(Set partitions); + + void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java new file mode 100644 index 0000000000..e8e8ee610e --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.processor.EdqsProducer; +import org.thingsboard.server.edqs.util.EdqsPartitionService; +import org.thingsboard.server.edqs.util.VersionsStore; +import org.thingsboard.server.gen.transport.TransportProtos.EdqsEventMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; +import org.thingsboard.server.queue.common.consumer.QueueConsumerManager; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.EdqsQueueFactory; +import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@RequiredArgsConstructor +@KafkaEdqsComponent +@Slf4j +public class KafkaEdqsStateService implements EdqsStateService { + + private final EdqsConfig config; + private final EdqsPartitionService partitionService; + private final EdqsQueueFactory queueFactory; + private final EdqsProcessor edqsProcessor; + + private MainQueueConsumerManager, QueueConfig> stateConsumer; + private QueueConsumerManager> eventsConsumer; + private EdqsProducer stateProducer; + + private ExecutorService consumersExecutor; + private ExecutorService mgmtExecutor; + private ScheduledExecutorService scheduler; + + private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger restoredCount = new AtomicInteger(); + + @PostConstruct + private void init() { + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-backup-consumer")); + mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); + scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); + + stateConsumer = MainQueueConsumerManager., QueueConfig>builder() + .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) + .config(QueueConfig.of(true, config.getPollInterval())) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + edqsProcessor.process(msg, EdqsQueue.STATE); + if (restoredCount.incrementAndGet() % 1000 == 0) { + log.info("Processed {} msgs", restoredCount.get()); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator((config, partitionId) -> queueFactory.createEdqsMsgConsumer(EdqsQueue.STATE)) + .consumerExecutor(consumersExecutor) + .taskExecutor(mgmtExecutor) + .scheduler(scheduler) + .build(); + + eventsConsumer = QueueConsumerManager.>builder() + .name("edqs-events-to-backup-consumer") + .pollInterval(config.getPollInterval()) + .msgPackProcessor((msgs, consumer) -> { + for (TbProtoQueueMsg queueMsg : msgs) { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + return; + } + } + + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } + } + consumer.commit(); + }) + .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group + .consumerExecutor(consumersExecutor) + .threadPrefix("edqs-events-to-backup") + .build(); + + stateProducer = EdqsProducer.builder() + .queue(EdqsQueue.STATE) + .partitionService(partitionService) + .producer(queueFactory.createEdqsMsgProducer(EdqsQueue.STATE)) + .build(); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + eventsConsumer.subscribe(); + eventsConsumer.launch(); + } + + @Override + public void restore(Set partitions) { + restoredCount.set(0); + long startTs = System.currentTimeMillis(); + log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); + + stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update + stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait + + log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), restoredCount.get()); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + // do nothing here, backup is done by events consumer + } + + private TenantId getTenantId(ToEdqsMsg edqsMsg) { + return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); + } + + @PreDestroy + private void preDestroy() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + eventsConsumer.stop(); + stateProducer.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java new file mode 100644 index 0000000000..8786f1866c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.edqs.processor.EdqsProcessor; +import org.thingsboard.server.edqs.util.EdqsRocksDb; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.util.Set; + +@Service +@RequiredArgsConstructor +@InMemoryEdqsComponent +@Slf4j +public class LocalEdqsStateService implements EdqsStateService { + + @Autowired @Lazy + private EdqsProcessor processor; + @Autowired + private EdqsRocksDb db; + + private Set partitions; + + @Override + public void restore(Set partitions) { + if (this.partitions == null) { + this.partitions = partitions; + } else { + return; + } + + db.forEach((key, value) -> { + try { + ToEdqsMsg edqsMsg = ToEdqsMsg.parseFrom(value); + log.trace("[{}] Restored msg from RocksDB: {}", key, edqsMsg); + processor.process(edqsMsg, EdqsQueue.STATE); + } catch (Exception e) { + log.error("[{}] Failed to restore value", key, e); + } + }); + } + + @Override + public void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg) { + log.trace("Save to RocksDB: {} {} {} {}", tenantId, type, key, msg); + try { + if (eventType == EdqsEventType.DELETED) { + db.delete(key); + } else { + db.put(key, msg.toByteArray()); + } + } catch (Exception e) { + log.error("[{}] Failed to save event {}", key, msg, e); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java new file mode 100644 index 0000000000..cfca43b08c --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.stats; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.queue.edqs.EdqsComponent; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +@EdqsComponent +@Service +@Slf4j +@RequiredArgsConstructor +@ConditionalOnProperty(name = "queue.edqs.stats.enabled", havingValue = "true", matchIfMissing = true) +public class EdqsStatsService { + + private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:60000}", + fixedDelayString = "${queue.edqs.stats.print-interval-ms:60000}") + private void reportStats() { + String values = statsMap.entrySet().stream() + .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") + .collect(Collectors.joining(System.lineSeparator())); + log.info("EDQS Stats: {}", values); + } + + public void reportTenantEdqsObject(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) + .reportEdqsObject(objectType, eventType); + } + + @Getter + @AllArgsConstructor + static class EdqsStats { + + private final TenantId tenantId; + private final ConcurrentHashMap entityCounters = new ConcurrentHashMap<>(); + private final StatsFactory statsFactory; + + private AtomicInteger getOrCreateObjectCounter(ObjectType objectType) { + return entityCounters.computeIfAbsent(objectType, + type -> statsFactory.createGauge(StatsType.EDQS.getName() + "_object_count", new AtomicInteger(), + "tenantId", tenantId.toString(), "objectType", type.name())); + } + + @Override + public String toString() { + return entityCounters.entrySet().stream() + .map(counters -> counters.getKey().name()+ " total = [" + counters.getValue() + "]") + .collect(Collectors.joining(", ")); + } + + public void reportEdqsObject(ObjectType objectType, EdqsEventType eventType) { + AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); + if (eventType == EdqsEventType.UPDATED){ + objectCounter.incrementAndGet(); + } else if (eventType == EdqsEventType.DELETED) { + objectCounter.decrementAndGet(); + } + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java new file mode 100644 index 0000000000..e8c260a996 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsPartitionService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsConfig.EdqsPartitioningStrategy; + +@Service +@RequiredArgsConstructor +public class EdqsPartitionService { + + private final HashPartitionService hashPartitionService; + private final EdqsConfig edqsConfig; + + public Integer resolvePartition(TenantId tenantId) { + if (edqsConfig.getPartitioningStrategy() == EdqsPartitioningStrategy.TENANT) { + return hashPartitionService.resolvePartitionIndex(tenantId.getId(), edqsConfig.getPartitions()); + } else { + return null; + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java new file mode 100644 index 0000000000..933d293c79 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsRocksDb.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.Getter; +import org.rocksdb.Options; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.edqs.InMemoryEdqsComponent; + +import java.nio.file.Files; +import java.nio.file.Path; + +@Component +@InMemoryEdqsComponent +public class EdqsRocksDb extends TbRocksDb { + + @Getter + private boolean isNew; + + public EdqsRocksDb(@Value("${queue.edqs.local.rocksdb_path:${java.io.tmpdir}/edqs-backup}") String path) { + super(path, new Options().setCreateIfMissing(true)); + } + + @PostConstruct + public void init() { + isNew = !Files.exists(Path.of(path)); + super.init(); + } + + @PreDestroy + public void close() { + super.close(); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java new file mode 100644 index 0000000000..1ad2634c1b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -0,0 +1,424 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.permission.MergedGroupTypePermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.util.CollectionsUtil; +import org.thingsboard.server.edqs.data.EntityData; +import org.thingsboard.server.edqs.data.dp.DataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsCountQuery; +import org.thingsboard.server.edqs.query.EdqsDataQuery; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.query.EdqsQuery; +import org.thingsboard.server.edqs.query.SortableEntityData; +import org.thingsboard.server.edqs.repo.KeyDictionary; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.apache.commons.lang3.StringUtils.containsIgnoreCase; +import static org.thingsboard.server.common.data.StringUtils.equalsAny; +import static org.thingsboard.server.common.data.StringUtils.splitByCommaWithoutQuotes; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.AND; +import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation.OR; + +@Slf4j +public class RepositoryUtils { + + public static final Comparator SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue) + .thenComparing(sp -> sp.getId().toString()); + + public static final Comparator SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue) + .thenComparing(sp -> sp.getId().toString()).reversed(); + + public static final MergedUserPermissions SYS_ADMIN_PERMISSIONS = new MergedUserPermissions(Collections.singletonMap(Resource.ALL, Set.of(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY)), Collections.emptyMap()); + public static final MergedUserPermissions ALL_READ_PERMISSIONS = new MergedUserPermissions( + Collections.singletonMap(Resource.ALL, Set.of(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY)), Collections.emptyMap()); + + public static EntityType resolveEntityType(EntityFilter entityFilter) { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> ((SingleEntityFilter) entityFilter).getSingleEntity().getEntityType(); + case ENTITY_GROUP -> ((EntityGroupFilter) entityFilter).getGroupType(); + case ENTITY_LIST -> ((EntityListFilter) entityFilter).getEntityType(); + case ENTITY_NAME -> ((EntityNameFilter) entityFilter).getEntityType(); + case ENTITY_TYPE -> ((EntityTypeFilter) entityFilter).getEntityType(); + case ENTITY_GROUP_LIST, ENTITY_GROUP_NAME -> EntityType.ENTITY_GROUP; + case ENTITIES_BY_GROUP_NAME -> ((EntitiesByGroupNameFilter) entityFilter).getGroupType(); + case STATE_ENTITY_OWNER -> throw new RuntimeException("Not implemented!"); // TODO: implement + case ASSET_TYPE, ASSET_SEARCH_QUERY -> EntityType.ASSET; + case DEVICE_TYPE, DEVICE_SEARCH_QUERY -> EntityType.DEVICE; + case ENTITY_VIEW_TYPE, ENTITY_VIEW_SEARCH_QUERY -> EntityType.ENTITY_VIEW; + case EDGE_TYPE, EDGE_SEARCH_QUERY -> EntityType.EDGE; + case RELATIONS_QUERY -> { + RelationsQueryFilter rgf = (RelationsQueryFilter) entityFilter; + yield rgf.isMultiRoot() ? rgf.getMultiRootEntitiesType() : rgf.getRootEntity().getEntityType(); + } + case API_USAGE_STATE -> EntityType.API_USAGE_STATE; + case SCHEDULER_EVENT -> EntityType.SCHEDULER_EVENT; + }; + } + + public static boolean hasNoPermissionsForAllRelationQueryResources(Map permissionsMap) { + if (permissionsMap.get(Resource.resourceFromEntityType(EntityType.TENANT)).isHasGenericRead()) { + return false; + } + for (EntityType entityType : EntityGroup.groupTypes) { + if (permissionsMap.get(Resource.resourceFromEntityType(entityType)).isHasGenericRead() || + !permissionsMap.get(Resource.resourceFromEntityType(entityType)).getEntityGroupIds().isEmpty()) { + return false; + } + if (permissionsMap.get(Resource.groupResourceFromGroupType(entityType)).isHasGenericRead() || + !permissionsMap.get(Resource.groupResourceFromGroupType(entityType)).getEntityGroupIds().isEmpty()) { + return false; + } + } + return true; + } + + public static boolean customerUserIsTryingToAccessTenantEntity(QueryContext ctx, EntityFilter entityFilter) { + if (ctx.isTenantUser()) { + return false; + } else { + return switch (entityFilter.getType()) { + case SINGLE_ENTITY -> { + SingleEntityFilter seFilter = (SingleEntityFilter) entityFilter; + yield isSystemOrTenantEntity(seFilter.getSingleEntity().getEntityType()); + } + case ENTITY_LIST -> { + EntityListFilter elFilter = (EntityListFilter) entityFilter; + yield isSystemOrTenantEntity(elFilter.getEntityType()); + } + case ENTITY_NAME -> { + EntityNameFilter enFilter = (EntityNameFilter) entityFilter; + yield isSystemOrTenantEntity(enFilter.getEntityType()); + } + case ENTITY_TYPE -> { + EntityTypeFilter etFilter = (EntityTypeFilter) entityFilter; + yield isSystemOrTenantEntity(etFilter.getEntityType()); + } + default -> false; + }; + } + } + + private static boolean isSystemOrTenantEntity(EntityType entityType) { + return switch (entityType) { + case INTEGRATION, CONVERTER, DEVICE_PROFILE, ASSET_PROFILE, RULE_CHAIN, SCHEDULER_EVENT, TENANT, + TENANT_PROFILE, WIDGET_TYPE, WIDGETS_BUNDLE -> true; + default -> false; + }; + } + + public static EdqsDataQuery toNewQuery(EntityDataQuery oldQuery) { + var query = EdqsDataQuery.builder(); + query.page(oldQuery.getPageLink().getPage()); + query.pageSize(oldQuery.getPageLink().getPageSize()); + query.textSearch(oldQuery.getPageLink().getTextSearch()); + var sortOrder = oldQuery.getPageLink().getSortOrder(); + if (sortOrder != null && toNewKey(sortOrder.getKey()) != null) { + query.sortKey(toNewKey(sortOrder.getKey())); + query.sortDirection(sortOrder.getDirection()); + } else { + query.sortKey(new DataKey(EntityKeyType.ENTITY_FIELD, "createdTime", null)); + query.sortDirection(EntityDataSortOrder.Direction.DESC); + } + query.entityFilter(oldQuery.getEntityFilter()); + query.keyFilters(toKeyFilters(oldQuery.getKeyFilters())); + query.entityFields(toNewKeys(oldQuery.getEntityFields())); + query.latestValues(toNewKeys(oldQuery.getLatestValues())); + return query.build(); + } + + public static EdqsCountQuery toNewQuery(EntityCountQuery oldQuery) { + return EdqsCountQuery.builder() + .entityFilter(oldQuery.getEntityFilter()) + .hasKeyFilters(CollectionsUtil.isNotEmpty(oldQuery.getKeyFilters())) + .keyFilters(toKeyFilters(oldQuery.getKeyFilters())) + .build(); + } + + private static List toKeyFilters(List keyFilters) { + if (keyFilters == null || keyFilters.isEmpty()) { + return Collections.emptyList(); + } else { + List result = new ArrayList<>(); + for (KeyFilter entityFilter : keyFilters) { + var newKey = toNewKey(entityFilter.getKey()); + if (newKey != null) { + result.add(new EdqsFilter(newKey, entityFilter.getValueType(), entityFilter.getPredicate())); + } + } + return result; + } + } + + private static DataKey toNewKey(EntityKey entityKey) { + if (EntityKeyType.ENTITY_FIELD.equals(entityKey.getType())) { + return new DataKey(entityKey.getType(), entityKey.getKey(), null); + } + Integer keyId = KeyDictionary.get(entityKey.getKey()); + if (keyId != null) { + return new DataKey(entityKey.getType(), entityKey.getKey(), keyId); + } else { + log.warn("Missing dictionary key for {}", entityKey.getKey()); + return null; + } + } + + private static List toNewKeys(List entityKeys) { + if (entityKeys == null || entityKeys.isEmpty()) { + return Collections.emptyList(); + } else { + var result = new ArrayList(entityKeys.size()); + for (EntityKey entityKey : entityKeys) { + var newKey = toNewKey(entityKey); + if (newKey != null) { + result.add(newKey); + } + } + return result; + } + } + + public static boolean checkKeyFilters(EntityData entity, List keyFilters) { + for (EdqsFilter keyFilter : keyFilters) { + EntityKeyValueType valueType = keyFilter.valueType(); + if (valueType == null) { + valueType = switch (keyFilter.predicate().getType()) { + case STRING -> EntityKeyValueType.STRING; + case NUMERIC -> EntityKeyValueType.NUMERIC; + case BOOLEAN -> EntityKeyValueType.BOOLEAN; + default -> throw new IllegalStateException(); + }; + } + DataPoint dp = entity.getDataPoint(keyFilter.key(), null); + boolean checkResult = switch (valueType) { + case STRING -> { + String str = dp != null ? dp.valueToString() : null; + yield StringUtils.isEmpty(str) || checkKeyFilter(str, keyFilter.predicate()); + } + case BOOLEAN -> { + Boolean booleanValue = dp != null ? dp.getBool() : null; + yield booleanValue != null && checkKeyFilter(booleanValue, keyFilter.predicate()); + } + case DATE_TIME, NUMERIC -> { + Double doubleValue = dp != null ? dp.getDouble() : null; + yield doubleValue != null && checkKeyFilter(doubleValue, keyFilter.predicate()); + } + }; + if (!checkResult) { + return false; + } + } + return true; + } + + public static boolean checkKeyFilter(String value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.STRING) { + throw new IllegalStateException("Not implemented"); + } + StringFilterPredicate predicate = (StringFilterPredicate) keyFilterPredicate; + String predicateValue = predicate.getValue().getValue(); + if (StringUtils.isEmpty(predicateValue)) { + return true; + } + if (predicate.isIgnoreCase()) { + predicateValue = predicateValue.toLowerCase(); + value = value.toLowerCase(); + } + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case STARTS_WITH -> value.startsWith(predicateValue); + case ENDS_WITH -> value.endsWith(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case CONTAINS -> value.contains(predicateValue); + case NOT_CONTAINS -> !value.contains(predicateValue); + case IN -> equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + case NOT_IN -> !equalsAny(value, splitByCommaWithoutQuotes(predicateValue)); + }; + } + + public static boolean checkKeyFilter(Double value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.NUMERIC) { + throw new IllegalStateException("Not implemented"); + } + NumericFilterPredicate predicate = (NumericFilterPredicate) keyFilterPredicate; + Double predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + case GREATER -> value.compareTo(predicateValue) > 0; + case LESS -> value.compareTo(predicateValue) < 0; + case GREATER_OR_EQUAL -> value.compareTo(predicateValue) >= 0; + case LESS_OR_EQUAL -> value.compareTo(predicateValue) <= 0; + }; + } + + public static boolean checkKeyFilter(Boolean value, KeyFilterPredicate keyFilterPredicate) { + if (keyFilterPredicate.getType() == FilterPredicateType.COMPLEX) { + return checkComplexKeyFilter(value, (ComplexFilterPredicate) keyFilterPredicate, RepositoryUtils::checkKeyFilter); + } + if (keyFilterPredicate.getType() != FilterPredicateType.BOOLEAN) { + throw new IllegalStateException("Not implemented"); + } + BooleanFilterPredicate predicate = (BooleanFilterPredicate) keyFilterPredicate; + Boolean predicateValue = predicate.getValue().getValue(); + return switch (predicate.getOperation()) { + case EQUAL -> value.equals(predicateValue); + case NOT_EQUAL -> !value.equals(predicateValue); + }; + } + + public static boolean checkComplexKeyFilter(T value, ComplexFilterPredicate filterPredicates, + SimpleKeyFilter simpleKeyFilter) { + if (filterPredicates.getOperation() == AND) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (!simpleKeyFilter.check(value, filterPredicate)) { + return false; + } + } + return true; + } else if (filterPredicates.getOperation() == OR) { + for (KeyFilterPredicate filterPredicate : filterPredicates.getPredicates()) { + if (simpleKeyFilter.check(value, filterPredicate)) { + return true; + } + } + return false; + } else { + return false; + } + } + + public static Pattern toSqlLikePattern(String nameFilter) { + if (StringUtils.isNotBlank(nameFilter)) { + boolean percentSymbolOnStart = nameFilter.startsWith("%"); + boolean percentSymbolOnEnd = nameFilter.endsWith("%"); + if (percentSymbolOnStart) { + nameFilter = nameFilter.substring(1); + } + if (percentSymbolOnEnd) { + nameFilter = nameFilter.substring(0, nameFilter.length() - 1); + } + if (percentSymbolOnStart || percentSymbolOnEnd) { + return Pattern.compile((percentSymbolOnStart ? ".*" : "") + Pattern.quote(nameFilter) + (percentSymbolOnEnd ? ".*" : ""), Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(Pattern.quote(nameFilter) + ".*", Pattern.CASE_INSENSITIVE); + } + } + return null; + } + + @FunctionalInterface + public interface SimpleKeyFilter { + + boolean check(T value, KeyFilterPredicate predicate); + + } + + public static TsValue toTsValue(long ts, DataPoint dp) { + if (dp != null) { + return new TsValue(dp.getTs() > 0 ? dp.getTs() : ts, dp.valueToString()); + } else { + return new TsValue(ts, ""); + } + } + + public static String getSortValue(EntityData entity, DataKey sortKey) { + if (sortKey == null) { + return null; + } + switch (sortKey.type()) { + case ENTITY_FIELD -> { + return entity.getField(sortKey.key()); + } + case ATTRIBUTE, CLIENT_ATTRIBUTE, SHARED_ATTRIBUTE, SERVER_ATTRIBUTE -> { + var dp = entity.getAttr(sortKey.keyId(), sortKey.type()); + return dp != null ? dp.valueToString() : ""; + } + case TIME_SERIES -> { + var dp = entity.getTs(sortKey.keyId()); + return dp != null ? dp.valueToString() : ""; + } + default -> throw new IllegalStateException("toSortKey is not implemented for type: " + sortKey.type()); + } + } + + public static boolean checkFilters(EdqsQuery query, EntityData entity) { + if (entity == null || entity.getFields() == null) { + return false; // Entity was already removed or not arrived yet; + } + if (query.isHasKeyFilters() && !checkKeyFilters(entity, query.getKeyFilters())) { + return false; + } + if (query instanceof EdqsDataQuery dataQuery) { + return !dataQuery.isHasTextSearch() || checkTextSearch(entity, dataQuery); + } + return true; + } + + private static boolean checkTextSearch(EntityData entityData, EdqsDataQuery query) { + return Stream.concat(query.getEntityFields().stream(), query.getLatestValues().stream()) + .anyMatch(key -> { + DataPoint value = entityData.getDataPoint(key, null); + return value != null && containsIgnoreCase(value.valueToString(), query.getTextSearch()); + }); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java new file mode 100644 index 0000000000..9b06a79a1f --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbRocksDb.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import java.nio.charset.StandardCharsets; +import java.util.function.BiConsumer; + +@RequiredArgsConstructor +public class TbRocksDb { + + protected final String path; + private final Options options; + + private RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + @SneakyThrows + public void init() { + db = RocksDB.open(options, path); + } + + public void put(String key, byte[] value) throws RocksDBException { + db.put(key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + public void delete(String key) throws RocksDBException { + db.delete(key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java new file mode 100644 index 0000000000..de2c5a1955 --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/VersionsStore.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class VersionsStore { + + private final Cache versions = Caffeine.newBuilder() + .expireAfterWrite(1, TimeUnit.HOURS) + .build(); + + public boolean isNew(String key, Long version) { + AtomicBoolean isNew = new AtomicBoolean(false); + versions.asMap().compute(key, (k, prevVersion) -> { + if (prevVersion == null || prevVersion < version) { + isNew.set(true); + return version; + } else { + if (version < prevVersion) { + log.info("[{}] Version {} is outdated, the latest is {}", key, version, prevVersion); + } + return prevVersion; + } + }); + return isNew.get(); + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java new file mode 100644 index 0000000000..9ca7d86e0a --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/edqs/EdqsService.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.edqs; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EdqsService { + + ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request); + + boolean isApiEnabled(); + + void onUpdate(TenantId tenantId, EntityId entityId, Object entity); + + void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void onDelete(TenantId tenantId, EntityId entityId); + + void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object); + + void processSystemRequest(ToCoreEdqsRequest request); + + void processSystemMsg(ToCoreEdqsMsg request); + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index f3a0e47d09..ce2d31a745 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -26,7 +26,8 @@ public enum ServiceType { TB_RULE_ENGINE("TB Rule Engine"), TB_TRANSPORT("TB Transport"), JS_EXECUTOR("JS Executor"), - TB_VC_EXECUTOR("TB VC Executor"); + TB_VC_EXECUTOR("TB VC Executor"), + EDQS("TB Entity Data Query Service"); private final String label; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java index cbddc83b2b..552bdf50d6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java @@ -30,26 +30,33 @@ public class TopicPartitionInfo { private final TenantId tenantId; private final Integer partition; @Getter + private final boolean useInternalPartition; + @Getter private final String fullTopicName; @Getter private final boolean myPartition; @Builder - public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean useInternalPartition, boolean myPartition) { this.topic = topic; this.tenantId = tenantId; this.partition = partition; + this.useInternalPartition = useInternalPartition; this.myPartition = myPartition; String tmp = topic; if (tenantId != null && !tenantId.isNullUid()) { tmp += ".isolated." + tenantId.getId().toString(); } - if (partition != null) { + if (partition != null && !useInternalPartition) { tmp += "." + partition; } this.fullTopicName = tmp; } + public TopicPartitionInfo(String topic, TenantId tenantId, Integer partition, boolean myPartition) { + this(topic, tenantId, partition, false, myPartition); + } + public TopicPartitionInfo newByTopic(String topic) { return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.myPartition); } @@ -66,6 +73,14 @@ public class TopicPartitionInfo { return Optional.ofNullable(partition); } + public TopicPartitionInfo withTopic(String topic) { + return new TopicPartitionInfo(topic, this.tenantId, this.partition, this.useInternalPartition, this.myPartition); + } + + public TopicPartitionInfo withUseInternalPartition(boolean useInternalPartition) { + return new TopicPartitionInfo(this.topic, this.tenantId, this.partition, useInternalPartition, this.myPartition); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -79,6 +94,7 @@ public class TopicPartitionInfo { @Override public int hashCode() { - return Objects.hash(fullTopicName); + return Objects.hash(fullTopicName, partition); } + } diff --git a/common/pom.xml b/common/pom.xml index 4c8f56cbce..9cb67ec989 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -49,6 +49,7 @@ edge-api version-control script + edqs diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 228a4039d2..d21862d3e1 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -70,6 +70,7 @@ message ServiceInfo { repeated string transports = 6; SystemInfoProto systemInfo = 10; repeated string assignedTenantProfiles = 11; + string label = 12; } message SystemInfoProto { @@ -170,12 +171,33 @@ message AttributeValueProto { optional int64 version = 10; } +message AttributeKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + AttributeScopeProto scope = 4; + string key = 5; + int64 version = 6; + int64 lastUpdateTs = 7; + KeyValueProto value = 8; +} + message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; optional int64 version = 3; } +message LatestTsKvProto { + int64 entityIdMSB = 1; + int64 entityIdLSB = 2; + EntityTypeProto entityType = 3; + string key = 4; + int64 ts = 5; + int64 version = 6; + KeyValueProto value = 7; +} + message TsKvListProto { int64 ts = 1; repeated KeyValueProto kv = 2; @@ -482,6 +504,10 @@ message ImageCacheKeyProto { optional string publicResourceKey = 2; } +message ToEdqsCoreServiceMsg { + bytes value = 1; +} + message LwM2MRegistrationRequestMsg { string tenantId = 1; string endpoint = 2; @@ -1532,6 +1558,7 @@ message ToCoreNotificationMsg { ToEdgeSyncRequestMsgProto toEdgeSyncRequest = 11 [deprecated = true]; FromEdgeSyncResponseMsgProto fromEdgeSyncResponse = 12 [deprecated = true]; ResourceCacheInvalidateMsg resourceCacheInvalidateMsg = 13; + ToEdqsCoreServiceMsg toEdqsCoreServiceMsg = 17; RestApiCallResponseMsgProto restApiCallResponseMsg = 50; } @@ -1660,3 +1687,33 @@ message HousekeeperTaskProto { int32 attempt = 50; repeated string errors = 51; } + +message ToEdqsMsg { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 customerIdMSB = 3; + int64 customerIdLSB = 4; + int64 ts = 5; + EdqsEventMsg eventMsg = 6; + EdqsRequestMsg requestMsg = 7; +} + +message FromEdqsMsg { + EdqsResponseMsg responseMsg = 1; +} + +message EdqsEventMsg { + string key = 1; + string objectType = 2; + bytes data = 3; + string eventType = 4; + optional int64 version = 5; +} + +message EdqsRequestMsg { + string value = 1; +} + +message EdqsResponseMsg { + string value = 1; +} diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 44e2239e05..99ad24575e 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -116,6 +116,10 @@ org.apache.curator curator-recipes + + org.xerial.snappy + snappy-java + org.springframework.boot spring-boot-starter-test diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 9513565ca1..88aa2a233f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -95,9 +95,8 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = getFullTopicNames(); - log.info("Subscribing to topics {}", topicNames); - doSubscribe(topicNames); + log.info("Subscribing to topics {}", getFullTopicNames()); + doSubscribe(partitions); subscribed = true; } records = partitions.isEmpty() ? emptyList() : doPoll(durationInMillis); @@ -187,7 +186,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(List topicNames); + abstract protected void doSubscribe(Set partitions); abstract protected void doCommit(); @@ -198,7 +197,10 @@ public abstract class AbstractTbQueueConsumerTemplate i if (partitions == null) { return Collections.emptyList(); } - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + return partitions.stream() + .map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? + "[" + tpi.getPartition().orElse(-1) + "]" : "")) + .collect(Collectors.toList()); } protected boolean isLongPollingSupported() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 950519b098..a2bdde6d66 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -211,6 +211,15 @@ public class DefaultTbQueueRequestTemplate send(Request request, long requestTimeoutNs) { + return send(request, requestTimeoutNs, null); + } + + @Override + public ListenableFuture send(Request request, Integer partition) { + return send(request, this.maxRequestTimeoutNs, partition); + } + + private ListenableFuture send(Request request, long requestTimeoutNs, Integer partition) { if (pendingRequests.mappingCount() >= maxPendingRequests) { log.warn("Pending request map is full [{}]! Consider to increase maxPendingRequests or increase processing performance. Request is {}", maxPendingRequests, request); return Futures.immediateFailedFuture(new RuntimeException("Pending request map is full!")); @@ -227,7 +236,7 @@ public class DefaultTbQueueRequestTemplate future, ResponseMetaData responseMetaData) { + void sendToRequestTemplate(Request request, UUID requestId, Integer partition, SettableFuture future, ResponseMetaData responseMetaData) { log.trace("[{}] Sending request, key [{}], expTime [{}], request {}", requestId, request.getKey(), responseMetaData.expTime, request); if (messagesStats != null) { messagesStats.incrementTotal(); } - requestTemplate.send(TopicPartitionInfo.builder().topic(requestTemplate.getDefaultTopic()).build(), request, new TbQueueCallback() { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .topic(requestTemplate.getDefaultTopic()) + .partition(partition) + .useInternalPartition(partition != null) + .build(); + requestTemplate.send(tpi, request, new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { if (messagesStats != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java index 1042eeb280..78f2085397 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueResponseTemplate.java @@ -28,6 +28,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueResponseTemplate; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -77,9 +78,18 @@ public class DefaultTbQueueResponseTemplate handler) { - this.responseTemplate.init(); + public void subscribe() { requestTemplate.subscribe(); + } + + @Override + public void subscribe(Set partitions) { + requestTemplate.subscribe(partitions); + } + + @Override + public void launch(TbQueueHandler handler) { + this.responseTemplate.init(); loopExecutor.submit(() -> { while (!stopped) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 6eb5c94c9b..2f44a9c4ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.consumer; +package org.thingsboard.server.queue.common.consumer; import lombok.Builder; import lombok.Getter; @@ -24,9 +24,6 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; -import org.thingsboard.server.service.queue.ruleengine.QueueEvent; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerManagerTask; -import org.thingsboard.server.service.queue.ruleengine.TbQueueConsumerTask; import java.util.Collection; import java.util.Collections; @@ -47,6 +44,7 @@ import java.util.stream.Collectors; @Slf4j public class MainQueueConsumerManager { + @Getter protected final QueueKey queueKey; @Getter protected C config; @@ -182,7 +180,7 @@ public class MainQueueConsumerManager partitions) { + public void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -236,13 +234,19 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } private static String partitionsToString(Collection partitions) { - return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.joining(", ", "[", "]")); + return partitions.stream().map(tpi -> tpi.getFullTopicName() + (tpi.isUseInternalPartition() ? + "[" + tpi.getPartition().orElse(-1) + "]" : "")) + .collect(Collectors.joining(", ", "[", "]")); } public interface MsgPackProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java index 9e5766b374..1a78cfc2ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/QueueEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/QueueEvent.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import java.io.Serializable; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java index e5821df68d..67bf370db0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerManagerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerManagerTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java rename to common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 5e672eb5c6..708e24fdac 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.queue.ruleengine; +package org.thingsboard.server.queue.common.consumer; import lombok.Getter; import lombok.Setter; @@ -70,10 +70,18 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(int timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { log.warn("[{}] Failed to await for consumer to stop", key, e); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 1ceb52b3d6..64677009ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.util.AfterContextReady; import java.net.InetAddress; @@ -63,6 +64,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.rule_engine.assigned_tenant_profiles:}") private Set assignedTenantProfiles; + @Autowired + private EdqsConfig edqsConfig; + @Autowired private ApplicationContext applicationContext; @@ -87,6 +91,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { assignedTenantProfiles = Collections.emptySet(); } + if (serviceTypes.contains(ServiceType.EDQS)) { + if (StringUtils.isBlank(edqsConfig.getLabel())) { + edqsConfig.setLabel(serviceId); + } + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -123,6 +132,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); } + builder.setLabel(edqsConfig.getLabel()); return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 37e519e3f2..165a54b300 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -20,6 +20,7 @@ import com.google.common.hash.Hashing; import jakarta.annotation.PostConstruct; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -70,6 +71,8 @@ public class HashPartitionService implements PartitionService { private String edgeTopic; @Value("${queue.edge.partitions:10}") private Integer edgePartitions; + @Value("${queue.edqs.partitions:12}") + private Integer edqsPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -123,6 +126,10 @@ public class HashPartitionService implements PartitionService { QueueKey edgeKey = coreKey.withQueueName(EDGE_QUEUE_NAME); partitionSizesMap.put(edgeKey, edgePartitions); partitionTopicsMap.put(edgeKey, edgeTopic); + + QueueKey edqsKey = new QueueKey(ServiceType.EDQS); + partitionSizesMap.put(edqsKey, edqsPartitions); + partitionTopicsMap.put(edqsKey, "edqs"); // placeholder, not used } @AfterStartUp(order = AfterStartUp.QUEUE_INFO_INITIALIZATION) @@ -211,7 +218,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -354,6 +361,11 @@ public class HashPartitionService implements PartitionService { } } + @Override + public boolean isSystemPartitionMine(ServiceType serviceType) { + return isMyPartition(serviceType, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); + } + @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { log.info("Recalculating partitions"); @@ -374,9 +386,9 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { try { - ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); - log.trace("Server responsible for {}[{}] - {}", queueKey, i, serviceInfo != null ? serviceInfo.getServiceId() : "none"); - if (currentService.equals(serviceInfo)) { + List services = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i, responsibleServices); + log.trace("Server responsible for {}[{}] - {}", queueKey, i, services); + if (services.contains(currentService)) { newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } catch (Exception e) { @@ -390,6 +402,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -410,19 +423,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -454,13 +464,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions, oldPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -468,6 +480,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return null; + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -598,6 +619,8 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(new QueueKey(serviceType).withQueueName(EDGE_QUEUE_NAME), key -> new ArrayList<>()).add(instance); } else if (ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); + } else if (ServiceType.EDQS.equals(serviceType)) { + queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } @@ -606,10 +629,11 @@ public class HashPartitionService implements PartitionService { } } - protected ServiceInfo resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, - Map> responsibleServices) { + @NotNull + protected List resolveByPartitionIdx(List servers, QueueKey queueKey, int partition, + Map> responsibleServices) { if (servers == null || servers.isEmpty()) { - return null; + return Collections.emptyList(); } TenantId tenantId = queueKey.getTenantId(); @@ -637,15 +661,21 @@ public class HashPartitionService implements PartitionService { responsibleServices.put(profileId, responsible); } if (responsible.isEmpty()) { - return null; + return Collections.emptyList(); } servers = responsible; } int hash = hash(tenantId.getId()); - return servers.get(Math.abs((hash + partition) % servers.size())); + ServiceInfo server = servers.get(Math.abs((hash + partition) % servers.size())); + return server != null ? List.of(server) : Collections.emptyList(); + } else if (queueKey.getType() == ServiceType.EDQS) { + List> sets = servers.stream().collect(Collectors.groupingBy(ServiceInfo::getLabel)) + .entrySet().stream().sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue).toList(); + return sets.get(partition % sets.size()); } else { - return servers.get(partition % servers.size()); + ServiceInfo server = servers.get(partition % servers.size()); + return server != null ? List.of(server) : Collections.emptyList(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b5744981bd..a95456b5e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -41,6 +41,8 @@ public interface PartitionService { boolean isMyPartition(ServiceType serviceType, TenantId tenantId, EntityId entityId); + boolean isSystemPartitionMine(ServiceType serviceType); + List getMyPartitions(QueueKey queueKey); /** @@ -61,8 +63,6 @@ public interface PartitionService { Set getOtherServices(ServiceType serviceType); - int resolvePartitionIndex(UUID entityId, int partitions); - void evictTenantInfo(TenantId tenantId); int countTransportsByType(String type); @@ -75,4 +75,6 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int resolvePartitionIndex(UUID entityId, int partitions); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 3a8d365433..8107b7c3eb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -19,6 +19,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ProtocolStringList; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -68,6 +69,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private Integer zkConnectionTimeout; @Value("${zk.session_timeout_ms}") private Integer zkSessionTimeout; + @Getter @Value("${zk.zk_dir}") private String zkDir; @Value("${zk.recalculate_delay:0}") @@ -80,6 +82,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private final PartitionService partitionService; private ScheduledExecutorService zkExecutorService; + @Getter private CuratorFramework client; private PathChildrenCache cache; private String nodePath; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 88ceb4aa08..e0e7db80a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; import java.io.Serial; +import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -36,12 +37,17 @@ public class PartitionChangeEvent extends TbApplicationEvent { @Getter private final ServiceType serviceType; @Getter - private final Map> partitionsMap; + private final Map> newPartitions; + @Getter + private final Map> oldPartitions; - public PartitionChangeEvent(Object source, ServiceType serviceType, Map> partitionsMap) { + public PartitionChangeEvent(Object source, ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { super(source); this.serviceType = serviceType; - this.partitionsMap = partitionsMap; + this.newPartitions = newPartitions; + this.oldPartitions = oldPartitions; } public Set getCorePartitions() { @@ -52,11 +58,16 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } + public Set getPartitions() { + return newPartitions.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + } + private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { - return partitionsMap.entrySet() + return newPartitions.entrySet() .stream() .filter(entry -> serviceType.equals(entry.getKey().getType()) && queueName.equals(entry.getKey().getQueueName())) .flatMap(entry -> entry.getValue().stream()) .collect(Collectors.toSet()); } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java new file mode 100644 index 0000000000..838f9b4aa2 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +// TODO: tb-core ? +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local'))") +public @interface EdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java new file mode 100644 index 0000000000..f35827a8ce --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Data +public class EdqsConfig { + + @Value("${queue.edqs.partitions:12}") + private int partitions; + @Value("${service.edqs.label:}") + private String label; + @Value("#{'${queue.edqs.partitioning_strategy:tenant}'.toUpperCase()}") + private EdqsPartitioningStrategy partitioningStrategy; + + @Value("${queue.edqs.requests_topic:edqs.requests}") + private String requestsTopic; + @Value("${queue.edqs.responses_topic:edqs.responses}") + private String responsesTopic; + @Value("${queue.edqs.poll_interval:125}") + private long pollInterval; + @Value("${queue.edqs.max_pending_requests:10000}") + private int maxPendingRequests; + @Value("${queue.edqs.max_request_timeout:10000}") + private int maxRequestTimeout; + + public String getLabel() { + if (partitioningStrategy == EdqsPartitioningStrategy.NONE) { + label = "all"; // single set for all instances, so that each instance has all partitions + } + return label; + } + + public enum EdqsPartitioningStrategy { + TENANT, NONE + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java new file mode 100644 index 0000000000..c773ea4e93 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueue.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.Getter; + +@Getter +public enum EdqsQueue { + + EVENTS("edqs.events", false, false), + STATE("edqs.state", true, true); + + private final String topic; + private final boolean readFromBeginning; // read from the beginning of the topic, instead of the latest committed offset + private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + + EdqsQueue(String topic, boolean readFromBeginning, boolean stopWhenRead) { + this.topic = topic; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java new file mode 100644 index 0000000000..dc4a9d645c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsQueueFactory.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface EdqsQueueFactory { + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue); + + TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group); + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java new file mode 100644 index 0000000000..5055787fde --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +public @interface InMemoryEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java new file mode 100644 index 0000000000..20b4beadcf --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsQueueFactory.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.memory.InMemoryStorage; +import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; +import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; + +@Component +@InMemoryEdqsComponent +@RequiredArgsConstructor +public class InMemoryEdqsQueueFactory implements EdqsQueueFactory { + + private final InMemoryStorage storage; + private final EdqsConfig edqsConfig; + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueConsumer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return createEdqsMsgConsumer(queue); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + if (queue == EdqsQueue.STATE) { + throw new UnsupportedOperationException(); + } + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + TbQueueConsumer> requestConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueProducer> responseProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getResponsesTopic()); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer) + .responseTemplate(responseProducer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(new DummyMessagesStats()) // FIXME + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java new file mode 100644 index 0000000000..9f112e7f59 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") +public @interface KafkaEdqsComponent { +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java new file mode 100644 index 0000000000..4b599670a1 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsQueueFactory.java @@ -0,0 +1,122 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.edqs; + +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.stats.DummyMessagesStats; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueResponseTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueResponseTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; + +import java.util.concurrent.atomic.AtomicInteger; + +@Component +@KafkaEdqsComponent +public class KafkaEdqsQueueFactory implements EdqsQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbKafkaAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; + private final TbKafkaAdmin edqsStateAdmin; + private final EdqsConfig edqsConfig; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final AtomicInteger consumerCounter = new AtomicInteger(); + + public KafkaEdqsQueueFactory(TbKafkaSettings kafkaSettings, TbKafkaTopicConfigs topicConfigs, + EdqsConfig edqsConfig, TbServiceInfoProvider serviceInfoProvider, + TbKafkaConsumerStatsService consumerStatsService) { + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsRequestsConfigs()); + this.edqsStateAdmin = new TbKafkaAdmin(kafkaSettings, topicConfigs.getEdqsStateConfigs()); + this.kafkaSettings = kafkaSettings; + this.edqsConfig = edqsConfig; + this.serviceInfoProvider = serviceInfoProvider; + this.consumerStatsService = consumerStatsService; + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue) { + String consumerGroup = "edqs-" + queue.name().toLowerCase() + "-consumer-group-" + serviceInfoProvider.getServiceId(); + return createEdqsMsgConsumer(queue, consumerGroup); + } + + @Override + public TbQueueConsumer> createEdqsMsgConsumer(EdqsQueue queue, String group) { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(queue.getTopic()) + .readFromBeginning(queue.isReadFromBeginning()) + .stopWhenRead(queue.isStopWhenRead()) + .clientId("edqs-" + queue.name().toLowerCase() + "-" + consumerCounter.getAndIncrement() + "-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(group) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-" + queue.name().toLowerCase() + "-producer-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(queue == EdqsQueue.STATE ? edqsStateAdmin : edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueResponseTemplate, TbProtoQueueMsg> createEdqsResponseTemplate() { + String requestsConsumerGroup = "edqs-requests-consumer-group-" + edqsConfig.getLabel(); + var requestConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(edqsConfig.getRequestsTopic()) + .clientId("edqs-requests-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(requestsConsumerGroup) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + var responseProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-response-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(edqsConfig.getResponsesTopic()) + .admin(edqsRequestsAdmin); + return DefaultTbQueueResponseTemplate., TbProtoQueueMsg>builder() + .requestTemplate(requestConsumer.build()) + .responseTemplate(responseProducer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .requestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .stats(new DummyMessagesStats()) // FIXME + .executor(ThingsBoardExecutors.newWorkStealingPool(5, "edqs")) + .build(); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java new file mode 100644 index 0000000000..b69b98dd4f --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLock.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLock { + + void lock(); + + void unlock(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java new file mode 100644 index 0000000000..61a21cca7c --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DistributedLockService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +public interface DistributedLockService { + + DistributedLock getLock(String key); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java new file mode 100644 index 0000000000..03b6d0f2d9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/DummyDistributedLockService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.util.concurrent.locks.ReentrantLock; + +@Service +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class DummyDistributedLockService implements DistributedLockService { + + @Override + public DistributedLock getLock(String key) { + return new DummyDistributedLock<>(); + } + + @RequiredArgsConstructor + private static class DummyDistributedLock implements DistributedLock { + + private final ReentrantLock lock = new ReentrantLock(); + + @SneakyThrows + @Override + public void lock() { + lock.lock(); + } + + @SneakyThrows + @Override + public void unlock() { + lock.unlock(); + } + + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java new file mode 100644 index 0000000000..07b8b503f9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/environment/ZkDistributedLockService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.environment; + +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.recipes.locks.InterProcessLock; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.ZkDiscoveryService; + +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "true") +@Slf4j +public class ZkDistributedLockService implements DistributedLockService { + + private final ZkDiscoveryService zkDiscoveryService; + + @Override + public DistributedLock getLock(String key) { + return new ZkDistributedLock<>(key); + } + + @RequiredArgsConstructor + private class ZkDistributedLock implements DistributedLock { + + private final InterProcessLock interProcessLock; + + public ZkDistributedLock(String key) { + this.interProcessLock = new InterProcessMutex(zkDiscoveryService.getClient(), zkDiscoveryService.getZkDir() + "/locks/" + key); + } + + @SneakyThrows + @Override + public void lock() { + interProcessLock.acquire(); + } + + @SneakyThrows + @Override + public void unlock() { + interProcessLock.release(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index a2edc35d94..2469e720bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -28,7 +28,13 @@ public class KafkaTbQueueMsg implements TbQueueMsg { private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + UUID key; + try { + key = UUID.fromString(record.key()); + } catch (IllegalArgumentException e) { + key = null; // FIXME + } + this.key = key; TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 2ea11c7afa..461defa58f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -57,7 +57,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { String numPartitionsStr = topicConfigs.get(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); if (numPartitionsStr != null) { numPartitions = Integer.parseInt(numPartitionsStr); - topicConfigs.remove("partitions"); } else { numPartitions = 1; } @@ -71,7 +70,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); + Map configs = PropertyUtils.getProps(topicConfigs, properties); + configs.remove(TbKafkaTopicConfigs.NUM_PARTITIONS_SETTING); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(configs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -188,6 +189,9 @@ public class TbKafkaAdmin implements TbQueueAdmin { public boolean isTopicEmpty(String topic) { try { + if (!getTopics().contains(topic)) { + return true; + } TopicDescription topicDescription = settings.getAdminClient().describeTopics(Collections.singletonList(topic)).topicNameValues().get(topic).get(); List partitions = topicDescription.partitions().stream().map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())).toList(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index ef79834735..167fbb1751 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -21,7 +21,9 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; @@ -30,8 +32,12 @@ import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +52,17 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached an empty msg pack + private Map endOffsets; // needed if stopWhenRead is true + + private boolean partitionsAssigned = false; + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +80,45 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override - protected void doSubscribe(List topicNames) { - if (!topicNames.isEmpty()) { - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + protected void doSubscribe(Set partitions) { + Map> topics; + if (partitions == null) { + topics = Collections.emptyMap(); + } else { + topics = new HashMap<>(); + partitions.forEach(tpi -> { + if (tpi.isUseInternalPartition()) { + topics.computeIfAbsent(tpi.getFullTopicName(), t -> new ArrayList<>()).add(tpi.getPartition().get()); + } else { + topics.put(tpi.getFullTopicName(), null); + } + }); + } + if (!topics.isEmpty()) { + topics.keySet().forEach(admin::createTopicIfNotExists); + List toSubscribe = new ArrayList<>(); + topics.forEach((topic, kafkaPartitions) -> { + if (kafkaPartitions == null) { + toSubscribe.add(topic); + } else { + consumer.assign(kafkaPartitions.stream() + .map(partition -> new TopicPartition(topic, partition)) + .toList()); + partitionsAssigned = true; + onPartitionsAssigned(); + } + }); + if (!toSubscribe.isEmpty()) { + consumer.subscribe(toSubscribe); + } + if (readFromBeginning) { + consumer.seekToBeginning(Collections.emptySet()); // for all assigned partitions + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -88,6 +133,13 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue log.trace("poll topic {} maxDuration {}", getTopic(), durationInMillis); ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (!partitionsAssigned) { + if (readFromBeginning) { + consumer.seekToBeginning(Collections.emptySet()); + } + partitionsAssigned = true; + onPartitionsAssigned(); + } stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); @@ -96,11 +148,36 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue return Collections.emptyList(); } else { List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead) { + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + if (endOffsets != null && endOffsets.isEmpty()) { + log.info("Reached end offsets for {}, stopping consumer", consumer.assignment()); + stop(); + } return recordList; } } + private void onPartitionsAssigned() { + if (stopWhenRead) { + endOffsets = consumer.endOffsets(consumer.assignment()).entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + @Override public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 3c9b85e925..6acf24b3f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -54,7 +54,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro private final TbQueueAdmin admin; - private final Set topics; + private final Set topics; @Getter private final String clientId; @@ -97,16 +97,21 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { - createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); + String topic = tpi.getFullTopicName(); + createTopicIfNotExist(topic); byte[] data = msg.getData(); ProducerRecord record; List

headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); if (log.isDebugEnabled()) { addAnalyticHeaders(headers); } - record = new ProducerRecord<>(tpi.getFullTopicName(), null, key, data, headers); + Integer partition = tpi.isUseInternalPartition() ? tpi.getPartition().orElse(null) : null; + record = new ProducerRecord<>(topic, partition, key, data, headers); producer.send(record, (metadata, exception) -> { if (exception == null) { if (callback != null) { @@ -130,12 +135,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro } } - private void createTopicIfNotExist(TopicPartitionInfo tpi) { - if (topics.contains(tpi)) { + private void createTopicIfNotExist(String topic) { + if (topics.contains(topic)) { return; } - admin.createTopicIfNotExists(tpi.getFullTopicName()); - topics.add(tpi); + admin.createTopicIfNotExists(topic); + topics.add(topic); } @Override @@ -144,4 +149,5 @@ public class TbKafkaProducerTemplate implements TbQueuePro producer.close(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index ee529e8a68..2c4ccae003 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -52,6 +52,12 @@ public class TbKafkaTopicConfigs { private String housekeeperProperties; @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") private String housekeeperReprocessingProperties; + @Value("${queue.kafka.topic-properties.edqs-events:}") + private String edqsEventsProperties; + @Value("${queue.kafka.topic-properties.edqs-requests:}") + private String edqsRequestsProperties; + @Value("${queue.kafka.topic-properties.edqs-state:}") + private String edqsStateProperties; @Getter private Map coreConfigs; @@ -79,6 +85,12 @@ public class TbKafkaTopicConfigs { private Map edgeConfigs; @Getter private Map edgeEventConfigs; + @Getter + private Map edqsEventsConfigs; + @Getter + private Map edqsRequestsConfigs; + @Getter + private Map edqsStateConfigs; @PostConstruct private void init() { @@ -97,6 +109,9 @@ public class TbKafkaTopicConfigs { housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); + edqsEventsConfigs = PropertyUtils.getProps(edqsEventsProperties); + edqsRequestsConfigs = PropertyUtils.getProps(edqsRequestsProperties); + edqsStateConfigs = PropertyUtils.getProps(edqsStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java new file mode 100644 index 0000000000..094f22c423 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/EdqsClientQueueFactory.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.edqs.EdqsQueue; + +public interface EdqsClientQueueFactory { + + TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue); + + TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index d70cad159b..a7c351a3c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; @@ -23,13 +24,19 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; +import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; @@ -43,37 +50,21 @@ import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") +@RequiredArgsConstructor public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TopicService topicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueAdmin queueAdmin; private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final EdqsConfig edqsConfig; private final InMemoryStorage storage; - public InMemoryMonolithQueueFactory(TopicService topicService, TbQueueCoreSettings coreSettings, - TbQueueRuleEngineSettings ruleEngineSettings, - TbQueueVersionControlSettings vcSettings, - TbServiceInfoProvider serviceInfoProvider, - TbQueueTransportApiSettings transportApiSettings, - TbQueueTransportNotificationSettings transportNotificationSettings, - TbQueueEdgeSettings edgeSettings, - InMemoryStorage storage) { - this.topicService = topicService; - this.coreSettings = coreSettings; - this.vcSettings = vcSettings; - this.serviceInfoProvider = serviceInfoProvider; - this.ruleEngineSettings = ruleEngineSettings; - this.transportApiSettings = transportApiSettings; - this.transportNotificationSettings = transportNotificationSettings; - this.edgeSettings = edgeSettings; - this.storage = storage; - } - @Override public TbQueueProducer> createTransportNotificationsMsgProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(transportNotificationSettings.getNotificationsTopic())); @@ -209,6 +200,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return null; } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return new InMemoryTbQueueProducer<>(storage, queue.getTopic()); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + TbQueueProducer> requestProducer = new InMemoryTbQueueProducer<>(storage, edqsConfig.getRequestsTopic()); + TbQueueConsumer> responseConsumer = new InMemoryTbQueueConsumer<>(storage, edqsConfig.getResponsesTopic()); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(queueAdmin) + .requestTemplate(requestProducer) + .responseTemplate(responseConsumer) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dd5d61e834..7a83264dfc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,11 +25,13 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -48,6 +50,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -80,6 +84,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueVersionControlSettings vcSettings; private final TbQueueEdgeSettings edgeSettings; private final TbKafkaConsumerStatsService consumerStatsService; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbKafkaAdmin ruleEngineAdmin; @@ -94,6 +99,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -107,7 +114,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueVersionControlSettings vcSettings, TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, - TbKafkaTopicConfigs kafkaTopicConfigs) { + TbKafkaTopicConfigs kafkaTopicConfigs, + EdqsConfig edqsConfig) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -119,6 +127,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.edgeSettings = edgeSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +142,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -490,6 +501,42 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .groupId(topicService.buildTopicName("monolith-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -523,4 +570,5 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi edgeAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index cc0e044917..2ac607a346 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -24,11 +24,13 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -47,6 +49,8 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsConfig; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -79,6 +83,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueEdgeSettings edgeSettings; + private final EdqsConfig edqsConfig; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; @@ -93,6 +98,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; + private final TbKafkaAdmin edqsRequestsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -107,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueEdgeSettings edgeSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, + EdqsConfig edqsConfig, TbKafkaTopicConfigs kafkaTopicConfigs) { this.topicService = topicService; this.kafkaSettings = kafkaSettings; @@ -119,6 +127,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; this.edgeSettings = edgeSettings; + this.edqsConfig = edqsConfig; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs()); @@ -133,6 +142,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); + this.edqsRequestsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsRequestsConfigs()); } @Override @@ -439,6 +450,42 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + var requestProducer = TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("edqs-request-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(edqsConfig.getRequestsTopic())) + .admin(edqsRequestsAdmin); + + var responseConsumer = TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(edqsConfig.getResponsesTopic() + "." + serviceInfoProvider.getServiceId())) + .clientId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .groupId(topicService.buildTopicName("tb-core-edqs-response-consumer-" + serviceInfoProvider.getServiceId())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), FromEdqsMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(edqsRequestsAdmin) + .statsService(consumerStatsService); + + return DefaultTbQueueRequestTemplate., TbProtoQueueMsg>builder() + .queueAdmin(edqsRequestsAdmin) + .requestTemplate(requestProducer.build()) + .responseTemplate(responseConsumer.build()) + .maxPendingRequests(edqsConfig.getMaxPendingRequests()) + .maxRequestTimeout(edqsConfig.getMaxRequestTimeout()) + .pollInterval(edqsConfig.getPollInterval()) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -469,4 +516,5 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { vcAdmin.destroy(); } } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 87a1a69c2e..b681f1fc2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,11 +23,13 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeEventNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToEdqsMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -43,6 +45,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.TopicService; +import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -81,6 +84,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; + private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -111,6 +115,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); + this.edqsEventsAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdqsEventsConfigs()); } @Override @@ -293,6 +298,20 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .build(); } + @Override + public TbQueueProducer> createEdqsMsgProducer(EdqsQueue queue) { + return TbKafkaProducerTemplate.>builder() + .clientId("edqs-producer-" + queue.name().toLowerCase() + "-" + serviceInfoProvider.getServiceId()) + .settings(kafkaSettings) + .admin(edqsEventsAdmin) + .build(); + } + + @Override + public TbQueueRequestTemplate, TbProtoQueueMsg> createEdqsRequestTemplate() { + throw new UnsupportedOperationException(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index c4002f4d3e..ec47177f28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -42,7 +42,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ec31763baa..913bd99ea7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -76,7 +76,7 @@ public interface TbQueueProducerProvider { */ TbQueueProducer> getTbUsageStatsMsgProducer(); - /** + /** * Used to push messages to other instances of TB Core Service * * @return diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index c406aeb311..d58441c0ad 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -36,7 +36,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory, EdqsClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index f00cc2c103..5b120e42e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -43,9 +43,8 @@ public class PropertyUtils { } public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { - Map properties = defaultProperties; + Map properties = new HashMap<>(defaultProperties); if (StringUtils.isNotBlank(propertiesStr)) { - properties = new HashMap<>(properties); properties.putAll(parser.apply(propertiesStr)); } return properties; diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java index b7a3ae20d3..6e8c9640e3 100644 --- a/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java +++ b/common/queue/src/test/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplateTest.java @@ -145,19 +145,19 @@ public class DefaultTbQueueRequestTemplateTest { @Test public void givenMessages_whenSend_thenOK() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); final int msgCount = 10; for (int i = 0; i < msgCount; i++) { inst.send(getRequestMsgMock()); } assertThat(inst.pendingRequests.mappingCount(), equalTo((long) msgCount)); - verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times(msgCount)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @Test public void givenMessagesOverMaxPendingRequests_whenSend_thenImmediateFailedFutureForTheOfRequests() { - willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any()); + willDoNothing().given(inst).sendToRequestTemplate(any(), any(), any(), any(), any()); inst.init(); int msgOverflowCount = 10; for (int i = 0; i < inst.maxPendingRequests; i++) { @@ -167,7 +167,7 @@ public class DefaultTbQueueRequestTemplateTest { assertThat("max pending requests overflow", inst.send(getRequestMsgMock()).isDone(), is(true)); //overflow, immediate failed future } assertThat(inst.pendingRequests.mappingCount(), equalTo(inst.maxPendingRequests)); - verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any()); + verify(inst, times((int) inst.maxPendingRequests)).sendToRequestTemplate(any(), any(), any(), any(), any()); } @SuppressWarnings("unchecked") diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java new file mode 100644 index 0000000000..7847860dcd --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyMessagesStats.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +public class DummyMessagesStats implements MessagesStats { + @Override + public void incrementTotal(int amount) { + + } + + @Override + public void incrementSuccessful(int amount) { + + } + + @Override + public void incrementFailed(int amount) { + + } + + @Override + public int getTotal() { + return 0; + } + + @Override + public int getSuccessful() { + return 0; + } + + @Override + public int getFailed() { + return 0; + } + + @Override + public void reset() { + + } + +} diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 3833155c05..2217be8d23 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -22,7 +22,8 @@ public enum StatsType { JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"), HOUSEKEEPER("housekeeper"), - EDGE("edge"); + EDGE("edge"), + EDQS("edqs"); private final String name; diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index fe0e019537..8dc98377d3 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -559,6 +559,10 @@ public class JacksonUtil { } } + public static JsonNode getValueByPath(ObjectNode node, String path) { + return node.at("/" + path.replace(".", "/")); + } + @Data public static class JsonNodeProcessingTask { private final String path; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index ca0636761d..bde497ed94 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -175,7 +175,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } } - consumer.subscribe(event.getPartitionsMap().values().stream().findAny().orElse(Collections.emptySet())); + consumer.subscribe(event.getNewPartitions().values().stream().findAny().orElse(Collections.emptySet())); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index e912872496..79a4ced3b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -17,7 +17,11 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.util.TbPair; import java.util.Collection; import java.util.List; @@ -45,6 +49,12 @@ public interface Dao { List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); - default EntityType getEntityType() { return null; } + default PageData findAllFields(PageLink pageLink) { + throw new UnsupportedOperationException(); + } + + default EntityType getEntityType() { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 01a72ea58c..a4bf25cd4d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.dao.model.ToData; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -85,6 +86,10 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } + public static Pageable toPageable(PageLink pageLink, String... sortColumns) { + return toPageable(pageLink, Collections.emptyMap(), Arrays.stream(sortColumns).map(column -> new SortOrder(column, SortOrder.Direction.ASC)).toList(), false); + } + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return toPageable(pageLink, columnMap, sortOrders, true); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index d0c54550ba..19fe45ed88 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,9 +15,23 @@ */ package org.thingsboard.server.dao; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; -public interface TenantEntityDao { +public interface TenantEntityDao { + + default Long countByTenantId(TenantId tenantId) { + throw new UnsupportedOperationException(); + } + + default PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + throw new UnsupportedOperationException(); + } + + default ObjectType getType() { + throw new UnsupportedOperationException(); + } - Long countByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 42ab2e2545..ffc2ebe796 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. @@ -226,4 +226,5 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); PageData> getAllAssetTypes(PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7d4d5300f9..305bba4004 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -148,12 +148,11 @@ public class AssetProfileServiceImpl extends CachedVersionedEntityService findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); + PageData findAll(PageLink pageLink); + ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 0694178540..512ac5913d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; @@ -24,13 +26,18 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.Validator; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -45,16 +52,15 @@ import static org.thingsboard.server.dao.attributes.AttributeUtils.validate; @ConditionalOnProperty(prefix = "cache.attributes", value = "enabled", havingValue = "false", matchIfMissing = true) @Primary @Slf4j +@RequiredArgsConstructor public class BaseAttributesService implements AttributesService { + private final AttributesDao attributesDao; + private final EdqsService edqsService; @Value("${sql.attributes.value_no_xss_validation:false}") private boolean valueNoXssValidation; - public BaseAttributesService(AttributesDao attributesDao) { - this.attributesDao = attributesDao; - } - @Override public ListenableFuture> find(TenantId tenantId, EntityId entityId, AttributeScope scope, String attributeKey) { validate(entityId, scope); @@ -98,26 +104,51 @@ public class BaseAttributesService implements AttributesService { public ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { validate(entityId, scope); AttributeUtils.validate(attribute, valueNoXssValidation); - return attributesDao.save(tenantId, entityId, scope, attribute); + return doSave(tenantId, entityId, scope, attribute); } @Override public ListenableFuture> save(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributes) { validate(entityId, scope); AttributeUtils.validate(attributes, valueNoXssValidation); - List> saveFutures = attributes.stream().map(attribute -> attributesDao.save(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); + List> saveFutures = attributes.stream().map(attribute -> doSave(tenantId, entityId, scope, attribute)).collect(Collectors.toList()); return Futures.allAsList(saveFutures); } + private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { + ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); + return Futures.transform(future, version -> { + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attribute, version)); + return version; + }, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> removeAll(TenantId tenantId, EntityId entityId, AttributeScope scope, List attributeKeys) { validate(entityId, scope); - return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); + List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); + return Futures.transform(Futures.allAsList(futures), result -> { + List keys = new ArrayList<>(); + for (TbPair keyVersionPair : result) { + String key = keyVersionPair.getFirst(); + Long version = keyVersionPair.getSecond(); + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + keys.add(key); + } + return keys; + }, MoreExecutors.directExecutor()); } @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + deleted.forEach(attribute -> { + AttributeScope scope = attribute.getKey(); + String key = attribute.getValue(); + if (scope != null && key != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); + } + }); return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 1ebd5c0ba4..c5ac50c9f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -24,18 +24,22 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbCacheValueWrapper; import org.thingsboard.server.cache.VersionedTbCache; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; @@ -67,6 +71,7 @@ public class CachedAttributesService implements AttributesService { private final AttributesDao attributesDao; private final JpaExecutorService jpaExecutorService; private final CacheExecutorService cacheExecutorService; + private final EdqsService edqsService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; private final VersionedTbCache cache; @@ -79,11 +84,12 @@ public class CachedAttributesService implements AttributesService { public CachedAttributesService(AttributesDao attributesDao, JpaExecutorService jpaExecutorService, - StatsFactory statsFactory, + @Lazy EdqsService edqsService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, VersionedTbCache cache) { this.attributesDao = attributesDao; this.jpaExecutorService = jpaExecutorService; + this.edqsService = edqsService; this.cacheExecutorService = cacheExecutorService; this.cache = cache; @@ -237,8 +243,10 @@ public class CachedAttributesService implements AttributesService { private ListenableFuture doSave(TenantId tenantId, EntityId entityId, AttributeScope scope, AttributeKvEntry attribute) { ListenableFuture future = attributesDao.save(tenantId, entityId, scope, attribute); - return Futures.transform(future, version -> { - put(entityId, scope, new BaseAttributeKvEntry(((BaseAttributeKvEntry)attribute).getKv(), attribute.getLastUpdateTs(), version)); + return Futures.transform(future, version -> { + BaseAttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(((BaseAttributeKvEntry) attribute).getKv(), attribute.getLastUpdateTs(), version); + put(entityId, scope, attributeKvEntry); + edqsService.onUpdate(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, attributeKvEntry, version)); return version; }, cacheExecutor); } @@ -256,7 +264,9 @@ public class CachedAttributesService implements AttributesService { List>> futures = attributesDao.removeAllWithVersions(tenantId, entityId, scope, attributeKeys); return Futures.allAsList(futures.stream().map(future -> Futures.transform(future, keyVersionPair -> { String key = keyVersionPair.getFirst(); - cache.evict(new AttributeCacheKey(scope, entityId, key), keyVersionPair.getSecond()); + Long version = keyVersionPair.getSecond(); + cache.evict(new AttributeCacheKey(scope, entityId, key), version); + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); return key; }, cacheExecutor)).collect(Collectors.toList())); } @@ -269,6 +279,8 @@ public class CachedAttributesService implements AttributesService { String key = deleted.getValue(); if (scope != null && key != null) { cache.evict(new AttributeCacheKey(scope, entityId, key)); + // FIXME: version is Long.MAX_VALUE because we expect that the entity is deleted and there won't be any attributes after this + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, Long.MAX_VALUE)); } }); return result.size(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index 7888161cd0..da050bdbc6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 2878fbd8e3..bc6adbe8b6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index 2e9e37577c..127abb5c3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -172,7 +172,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb var saved = dashboardDao.save(tenantId, dashboard); publishEvictEvent(new DashboardTitleEvictEvent(saved.getId())); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId) - .entityId(saved.getId()).created(dashboard.getId() == null).build()); + .entityId(saved.getId()).entity(saved).created(dashboard.getId() == null).build()); if (dashboard.getId() == null) { countService.publishCountEntityEvictEvent(tenantId, EntityType.DASHBOARD); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 2305e4419c..895eba06a5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -39,7 +39,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 09e2be01e0..b444ef16ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -180,13 +180,12 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService findAll(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java index d813d2e3c9..af79d0469a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeDao.java @@ -35,7 +35,7 @@ import java.util.UUID; * The Interface EdgeDao. * */ -public interface EdgeDao extends Dao, TenantEntityDao { +public interface EdgeDao extends Dao, TenantEntityDao { Edge save(TenantId tenantId, Edge edge); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3cdcf5e874..9fcb6c92ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.HasLabel; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTitle; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -40,9 +42,21 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.msg.edqs.EdqsService; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.sql.alarm.AlarmRepository; +import org.thingsboard.server.dao.sql.query.EntityMapping; +import org.thingsboard.server.dao.user.UserService; import java.util.ArrayList; import java.util.Collections; @@ -50,9 +64,12 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_GROUP_NAME; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_TYPE; import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; import static org.thingsboard.server.dao.service.Validator.validateEntityDataPageLink; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -79,12 +96,24 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy EntityServiceRegistry entityServiceRegistry; + @Autowired @Lazy + private EdqsService edqsService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); + + if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + EdqsRequest request = EdqsRequest.builder() + .entityCountQuery(query) + .userPermissions(userPermissions) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityCountQueryResult(); + } return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } @@ -95,6 +124,15 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); + if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + EdqsRequest request = EdqsRequest.builder() + .entityDataQuery(query) + .userPermissions(userPermissions) + .build(); + EdqsResponse response = processEdqsRequest(tenantId, customerId, request); + return response.getEntityDataQueryResult(); + } + if (!isValidForOptimization(query)) { return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); } @@ -110,6 +148,25 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); } + private boolean validForEdqs(EntityCountQuery query) { + return !(query.getEntityFilter() instanceof StateEntityOwnerFilter filter) || !EntityType.ALARM.equals(filter.getSingleEntity().getEntityType()); + } + + private EdqsResponse processEdqsRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + EdqsResponse response; + try { + log.info("Sending request to EDQS: {}", request); + response = edqsService.processRequest(tenantId, customerId, request).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + log.info("Received response from EDQS: {}", response); + if (response.getError() != null) { + throw new RuntimeException(response.getError()); + } + return response; + } + @Override public Optional fetchEntityName(TenantId tenantId, EntityId entityId) { log.trace("Executing fetchEntityName [{}]", entityId); @@ -184,6 +241,10 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe throw new IncorrectParameterException("Query entity filter type must be specified."); } else if (query.getEntityFilter().getType().equals(EntityFilterType.RELATIONS_QUERY)) { validateRelationQuery((RelationsQueryFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_TYPE)) { + validateEntityTypeQuery((EntityTypeFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_GROUP_NAME)) { + validateGroupNameQuery((EntityGroupNameFilter) query.getEntityFilter()); } } @@ -192,6 +253,18 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateEntityDataPageLink(query.getPageLink()); } + private static void validateEntityTypeQuery(EntityTypeFilter filter) { + if (filter.getEntityType() == null) { + throw new IncorrectParameterException("Entity type is required"); + } + } + + private static void validateGroupNameQuery(EntityGroupNameFilter filter) { + if (filter.getGroupType() == null) { + throw new IncorrectParameterException("Group type is required"); + } + } + private static void validateRelationQuery(RelationsQueryFilter queryFilter) { if (queryFilter.isMultiRoot() && queryFilter.getMultiRootEntitiesType() == null) { throw new IncorrectParameterException("Multi-root relation query filter should contain 'multiRootEntitiesType'"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 7b2f2d42c1..24f8aa4d77 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -18,7 +18,9 @@ package org.thingsboard.server.dao.entity; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.EnumMap; import java.util.List; @@ -26,26 +28,39 @@ import java.util.Map; @Service @Slf4j +@SuppressWarnings({"unchecked"}) public class EntityDaoRegistry { - private final Map> daos = new EnumMap<>(EntityType.class); + private final Map> tenantEntityDaos = new EnumMap<>(ObjectType.class); + private final Map> entityDaos = new EnumMap<>(EntityType.class); - private EntityDaoRegistry(List> daos) { - daos.forEach(dao -> { - EntityType entityType = dao.getEntityType(); - if (entityType != null) { - this.daos.put(entityType, dao); + private EntityDaoRegistry(List> entityDaos, List> tenantEntityDaos) { + entityDaos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.entityDaos.put(dao.getEntityType(), dao); + } + }); + tenantEntityDaos.forEach(dao -> { + if (dao.getType() != null) { + this.tenantEntityDaos.put(dao.getType(), dao); } }); } - @SuppressWarnings("unchecked") public Dao getDao(EntityType entityType) { - Dao dao = (Dao) daos.get(entityType); + Dao dao = (Dao) entityDaos.get(entityType); if (dao == null) { throw new IllegalArgumentException("Missing dao for entity type " + entityType); } return dao; } + public TenantEntityDao getTenantEntityDao(ObjectType objectType) { + TenantEntityDao dao = tenantEntityDaos.get(objectType); + if (dao == null) { + throw new IllegalArgumentException("Missing tenant entity dao for entity type " + objectType); + } + return (TenantEntityDao) dao; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 43b5e5cdfb..2dc3dab627 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -123,7 +123,7 @@ public class EntityViewServiceImpl extends CachedVersionedEntityService { private final T oldEntity; private final EntityId entityId; private final Boolean created; + private final Boolean broadcastEvent; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index d404c3128c..f762be0429 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -281,6 +281,8 @@ public class ModelConstants { public static final String ALARM_COMMENT_TYPE = "type"; public static final String ALARM_COMMENT_COMMENT = "comment"; + public static final String ALARM_TYPES_TABLE_NAME = "alarm_types"; + /** * Entity relation constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java new file mode 100644 index 0000000000..60f5232142 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AlarmTypeCompositeKey implements Serializable { + + private UUID tenantId; + private String type; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java new file mode 100644 index 0000000000..729d7a44f0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Table; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.ToData; + +import java.util.UUID; + +@Data +@Entity +@Table(name = ModelConstants.ALARM_TYPES_TABLE_NAME) +@IdClass(AlarmTypeCompositeKey.class) +public class AlarmTypeEntity implements ToData { + + @Id + @Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) + private UUID tenantId; + + @Id + @Column(name = ModelConstants.ALARM_TYPE_PROPERTY, nullable = false) + private String type; + + public AlarmTypeEntity() {} + + public AlarmTypeEntity(AlarmType alarmType) { + setTenantId(alarmType.getTenantId().getId()); + setType(alarmType.getType()); + } + + public AlarmTypeEntity(UUID tenantId, String type) { + this.tenantId = tenantId; + this.type = type; + } + + @Override + public AlarmType toData() { + AlarmType alarmType = new AlarmType(); + alarmType.setTenantId(TenantId.fromUUID(tenantId)); + alarmType.setType(type); + return alarmType; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java index 23fe580fa3..05930fbe00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sqlts/latest/TsKvLatestEntity.java @@ -26,6 +26,7 @@ import jakarta.persistence.SqlResultSetMapping; import jakarta.persistence.SqlResultSetMappings; import jakarta.persistence.Table; import lombok.Data; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; @@ -91,4 +92,12 @@ public final class TsKvLatestEntity extends AbstractTsKvEntity { this.strKey = strKey; this.version = version; } + + @Override + public TsKvEntry toData() { + TsKvEntry tsKvEntry = super.toData(); + tsKvEntry.setVersion(version); + return tsKvEntry; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java index 94126ebd1b..d92d7cfd0f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTargetDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; -public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface NotificationTargetDao extends Dao, TenantEntityDao, ExportableEntityDao { PageData findByTenantIdAndPageLink(TenantId tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index e83650b3b5..742ad45ffa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,5 +21,7 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityWithDataDao; public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { + Long sumDataSizeByTenantId(TenantId tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 69c6a7c418..efc9dab41b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -51,7 +53,10 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public QueueStats save(TenantId tenantId, QueueStats queueStats) { log.trace("Executing save [{}]", queueStats); queueStatsValidator.validate(queueStats, QueueStats::getTenantId); - return queueStatsDao.save(tenantId, queueStats); + QueueStats savedQueueStats = queueStatsDao.save(tenantId, queueStats); + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedQueueStats.getTenantId()).entityId(savedQueueStats.getId()) + .entity(savedQueueStats).created(queueStats.getId() == null).build()); + return savedQueueStats; } @Override @@ -93,6 +98,7 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu @Override public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { queueStatsDao.removeById(tenantId, id.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index d668f51dbd..d29b57c5c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.relation; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -41,6 +43,8 @@ public interface RelationDao { List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); + PageData findAll(PageLink pageLink); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 0e51cf256a..58cfdfe02a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -124,10 +124,9 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC if (ruleChain.getId() == null) { entityCountService.publishCountEntityEvictEvent(ruleChain.getTenantId(), EntityType.RULE_CHAIN); } - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) - .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedRuleChain.getTenantId()) + .entity(savedRuleChain).entityId(savedRuleChain.getId()).created(ruleChain.getId() == null) + .broadcastEvent(publishSaveEvent).build()); return savedRuleChain; } catch (Exception e) { checkConstraintViolation(e, "rule_chain_external_id_unq_key", "Rule Chain with such external id already exists!"); @@ -289,9 +288,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC relationService.saveRelations(tenantId, relations); } ruleChain = ruleChainDao.save(tenantId, ruleChain); - if (publishSaveEvent) { - eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain).entityId(ruleChain.getId()).build()); - } + eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId).entity(ruleChain) + .entityId(ruleChain.getId()).broadcastEvent(publishSaveEvent).build()); return RuleChainUpdateResult.successful(updatedRuleNodes); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index a394255b2e..0a8d449afe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -31,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java index d0faee3c9c..991147cedb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java @@ -35,5 +35,9 @@ public interface AlarmCommentRepository extends JpaRepository findAllByAlarmId(@Param("alarmId") UUID alarmId, - Pageable pageable); + Pageable pageable); + + @Query("SELECT c FROM AlarmCommentEntity c WHERE c.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 6083dac6cf..9f57ab6e52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -414,4 +414,6 @@ public interface AlarmRepository extends JpaRepository { @Param("alarmSeverities") List alarmSeverities, int limit); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java new file mode 100644 index 0000000000..b84542e3c3 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.dao.model.sql.AlarmTypeCompositeKey; +import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; + +import java.util.UUID; + +public interface AlarmTypeRepository extends JpaRepository { + + Page findByTenantId(UUID tenantId, Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index ccf82e1b5b..41e6f3dcc5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.alarm; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -42,4 +44,6 @@ public interface EntityAlarmRepository extends JpaRepository findAllByEntityId(UUID entityId); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 3d366a5a56..9d74b9bf8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCommentInfo; import org.thingsboard.server.common.data.id.AlarmId; @@ -29,6 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmCommentDao; import org.thingsboard.server.dao.model.sql.AlarmCommentEntity; import org.thingsboard.server.dao.sql.JpaPartitionedAbstractDao; @@ -44,7 +46,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ALARM_COMMENT_TABL @Component @SqlDao @RequiredArgsConstructor -public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao { +public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements AlarmCommentDao, TenantEntityDao { private final SqlPartitioningRepository partitioningRepository; @Value("${sql.alarm_comments.partition_size:168}") private int partitionSizeInHours; @@ -76,6 +78,11 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmCommentRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return AlarmCommentEntity.class; @@ -85,4 +92,10 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao getRepository() { return alarmCommentRepository; } + + @Override + public ObjectType getType() { + return ObjectType.ALARM_COMMENT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index e433dfbd86..7aea2adbec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -29,6 +29,7 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -57,6 +58,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AlarmEntity; @@ -84,7 +86,7 @@ import static org.thingsboard.server.dao.DaoUtil.toPageable; @Slf4j @Component @SqlDao -public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { +public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao, TenantEntityDao { @Autowired private AlarmRepository alarmRepository; @@ -551,9 +553,19 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ALARM; } + @Override + public ObjectType getType() { + return ObjectType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java new file mode 100644 index 0000000000..532f06eb03 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaAlarmTypeDao implements TenantEntityDao { + + @Autowired + private AlarmTypeRepository alarmTypeRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmTypeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "tenantId", "type"))); + } + + @Override + public ObjectType getType() { + return ObjectType.ALARM_TYPE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java new file mode 100644 index 0000000000..6bbc59559a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaEntityAlarmDao implements TenantEntityDao { + + @Autowired + private EntityAlarmRepository entityAlarmRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); + } + + @Override + public ObjectType getType() { + return ObjectType.ENTITY_ALARM; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java index 372201bc47..bd3f35cf76 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; @@ -81,4 +82,8 @@ public interface AssetProfileRepository extends JpaRepository findAllTenantAssetProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetProfileFields(a.id, a.createdTime, a.tenantId," + + "a.name, a.version, a.isDefault) FROM AssetProfileEntity a") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index bf4798fa4d..1be4fc6075 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; @@ -216,4 +217,9 @@ public interface AssetRepository extends JpaRepository, Expor @Query(value = "SELECT DISTINCT new org.thingsboard.server.common.data.util.TbPair(a.tenantId , a.type) FROM AssetEntity a") Page> getAllAssetTypes(Pageable pageable); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.AssetFields(a.id, a.createdTime, a.tenantId, a.customerId," + + "a.name, a.version, a.type, a.label, a.assetProfileId, a.additionalInfo) FROM AssetEntity a") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 0ef4370f30..569c13f376 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -22,8 +22,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -267,9 +269,24 @@ public class JpaAssetDao extends JpaAbstractDao implements A .map(AssetId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(assetRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; } + @Override + public ObjectType getType() { + return ObjectType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index 6ed74a5aa9..eb2fb6a9ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -21,13 +21,16 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; +import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.asset.AssetProfileDao; import org.thingsboard.server.dao.model.sql.AssetProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -37,7 +40,7 @@ import java.util.Optional; import java.util.UUID; @Component -public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao { +public class JpaAssetProfileDao extends JpaAbstractDao implements AssetProfileDao, TenantEntityDao { @Autowired private AssetProfileRepository assetProfileRepository; @@ -138,9 +141,24 @@ public class JpaAssetProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(assetProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ASSET_PROFILE; } + @Override + public ObjectType getType() { + return ObjectType.ASSET_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index a0e1f1447d..b344a0ca41 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AttributeScope; @@ -30,6 +31,8 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -48,6 +51,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -152,6 +156,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } + @Override + public PageData findAll(PageLink pageLink) { + Page attributes = attributeKvRepository.findAll(DaoUtil.toPageable(pageLink)); + return DaoUtil.pageToPageData(attributes); + } + @Override public List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId) { if (deviceProfileId != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 448b850ba7..19b3ec8fe8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -20,6 +20,8 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; @@ -55,4 +57,8 @@ public interface CustomerRepository extends JpaRepository, nativeQuery = true) Page findCustomersWithTheSameTitle(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.CustomerFields(c.id, c.createdTime, c.tenantId, " + + "c.title, c.version, c.additionalInfo, c.country, c.state, c.city, c.address, c.address2, c.zip, c.phone, c.email) FROM CustomerEntity c") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 497a2917d8..a371803763 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -20,6 +20,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.CustomerFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -104,9 +106,24 @@ public class JpaCustomerDao extends JpaAbstractDao imp ); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(customerRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; } + @Override + public ObjectType getType() { + return ObjectType.CUSTOMER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 6f182351c1..9932819ce0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -46,4 +47,8 @@ public interface DashboardRepository extends JpaRepository findAllIds(Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DashboardFields(d.id, d.createdTime, d.tenantId, " + + "d.customerId, d.title, d.version) FROM DashboardEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 640107add0..436c32d32b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -20,6 +20,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -90,9 +92,24 @@ public class JpaDashboardDao extends JpaAbstractDao return DaoUtil.pageToPageData(dashboardRepository.findAllIds(DaoUtil.toPageable(pageLink)).map(DashboardId::new)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(dashboardRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; } + @Override + public ObjectType getType() { + return ObjectType.DASHBOARD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index c136d361e2..f2b541cb45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.device; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -36,4 +38,8 @@ public interface DeviceCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index b51324f11a..e41796381d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -23,6 +23,8 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; @@ -92,4 +94,8 @@ public interface DeviceProfileRepository extends JpaRepository findAllTenantDeviceProfileNames(@Param("tenantId") UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields(d.id, d.createdTime, d.tenantId," + + "d.name, d.version, d.type, d.isDefault) FROM DeviceProfileEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c55210b606..161c201404 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -194,4 +195,9 @@ public interface DeviceRepository extends JpaRepository, Exp @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") UUID getExternalIdById(@Param("id") UUID id); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.DeviceFields(d.id, d.createdTime, d.tenantId, d.customerId," + + "d.name, d.version, d.type, d.label, d.deviceProfileId, d.additionalInfo) FROM DeviceEntity d") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 7571c00b6a..31d2aa6cb9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -19,10 +19,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -36,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { +public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao, TenantEntityDao { @Autowired private DeviceCredentialsRepository deviceCredentialsRepository; @@ -67,4 +71,14 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(deviceCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.DEVICE_CREDENTIALS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 48bb998016..2763e5dca7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,7 +29,9 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -282,9 +284,24 @@ public class JpaDeviceDao extends JpaAbstractDao implement .map(DeviceId::new).orElse(null); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(deviceRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; } + @Override + public ObjectType getType() { + return ObjectType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 4e1594eeff..ff6e61c051 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -24,12 +24,15 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -41,7 +44,7 @@ import java.util.UUID; @Component @SqlDao -public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao { +public class JpaDeviceProfileDao extends JpaAbstractDao implements DeviceProfileDao, TenantEntityDao { @Autowired private DeviceProfileRepository deviceProfileRepository; @@ -156,9 +159,24 @@ public class JpaDeviceProfileDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findDeviceProfiles(tenantId, pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(deviceProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; } + @Override + public ObjectType getType() { + return ObjectType.DEVICE_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java index 1bcfab7ab3..e75ae5984a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/EdgeRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.dao.model.sql.EdgeEntity; import org.thingsboard.server.dao.model.sql.EdgeInfoEntity; @@ -154,4 +155,8 @@ public interface EdgeRepository extends JpaRepository { EdgeEntity findByRoutingKey(String routingKey); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.EdgeFields(e.id, e.createdTime, e.tenantId, e.customerId," + + "e.name, e.version, e.type, e.label, e.additionalInfo) FROM EdgeEntity e") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 2249ced97c..eb46d5bec7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -22,8 +22,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; +import org.thingsboard.server.common.data.edqs.fields.EdgeFields; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -219,9 +221,24 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return edgeRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findEdgesByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(edgeRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.EDGE; } + @Override + public ObjectType getType() { + return ObjectType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 8f2e61d787..b4ad0478e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -145,4 +146,8 @@ public interface EntityViewRepository extends JpaRepository findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 98c5ad7036..82e7eab609 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -24,11 +24,14 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.GenericFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -47,8 +50,7 @@ import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; @Component @Slf4j @SqlDao -public class JpaEntityViewDao extends JpaAbstractDao - implements EntityViewDao { +public class JpaEntityViewDao extends JpaAbstractDao implements EntityViewDao, TenantEntityDao { @Autowired private EntityViewRepository entityViewRepository; @@ -218,8 +220,24 @@ public class JpaEntityViewDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(entityViewRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.ENTITY_VIEW; } + + @Override + public ObjectType getType() { + return ObjectType.ENTITY_VIEW; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java index 48a7df0f3b..e6df0ff235 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.NotificationRuleEntity; import org.thingsboard.server.dao.model.sql.NotificationRuleInfoEntity; import org.thingsboard.server.dao.notification.NotificationRuleDao; @@ -41,7 +43,7 @@ import java.util.UUID; @Component @SqlDao @RequiredArgsConstructor -public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao { +public class JpaNotificationRuleDao extends JpaAbstractDao implements NotificationRuleDao, TenantEntityDao { private final NotificationRuleRepository notificationRuleRepository; @@ -101,6 +103,11 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationRuleEntity.class; @@ -116,4 +123,9 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected Class getEntityClass() { return NotificationTargetEntity.class; @@ -116,4 +122,9 @@ public class JpaNotificationTargetDao extends JpaAbstractDao implements NotificationTemplateDao { +public class JpaNotificationTemplateDao extends JpaAbstractDao implements NotificationTemplateDao, TenantEntityDao { private final NotificationTemplateRepository notificationTemplateRepository; @@ -83,6 +85,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + @Override protected JpaRepository getRepository() { return notificationTemplateRepository; @@ -93,4 +100,9 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao implements OtaPackageDao { +public class JpaOtaPackageDao extends JpaAbstractDao implements OtaPackageDao, TenantEntityDao { @Autowired private OtaPackageRepository otaPackageRepository; @@ -52,9 +58,20 @@ public class JpaOtaPackageDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(otaPackageRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.OTA_PACKAGE; } + @Override + public ObjectType getType() { + return ObjectType.OTA_PACKAGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java index 707a6e7062..74f14a9697 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageInfoDao.java @@ -93,4 +93,5 @@ public class JpaOtaPackageInfoDao extends JpaAbstractDao { + @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); + + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 05da22f9a5..6c8f48446e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -28,6 +28,10 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.Resource; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataPageLink; @@ -128,7 +132,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { public PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds) { return transactionTemplate.execute(trStatus -> { AlarmDataPageLink pageLink = query.getPageLink(); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); ctx.addUuidListParameter("entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); StringBuilder selectPart = new StringBuilder(FIELDS_SELECTION); StringBuilder fromPart = new StringBuilder(" from alarm_info a "); @@ -315,7 +319,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { @Override public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); @@ -402,7 +406,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { }); } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && selectionMapping != null && !selectionMapping.isEmpty()) { String lowerSearchText = searchText.toLowerCase() + "%"; List searchPredicates = selectionMapping.stream() @@ -420,7 +424,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - private String buildPermissionsQuery(TenantId tenantId, QueryContext ctx) { + private String buildPermissionsQuery(TenantId tenantId, SqlQueryContext ctx) { StringBuilder permissionsQuery = new StringBuilder(); ctx.addUuidParameter("permissions_tenant_id", tenantId.getId()); permissionsQuery.append(" a.tenant_id = :permissions_tenant_id and ea.tenant_id = :permissions_tenant_id "); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 36ae779edd..31443f7f47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.QueryContext; import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; @@ -334,7 +335,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); @@ -416,7 +417,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, ignorePermissionCheck)); + SqlQueryContext ctx = new SqlQueryContext(new QueryContext(tenantId, customerId, entityType, ignorePermissionCheck)); EntityDataPageLink pageLink = query.getPageLink(); List mappings = EntityKeyMapping.prepareKeyMapping(entityType, query); @@ -524,7 +525,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { }); } - private String buildEntityWhere(QueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { + private String buildEntityWhere(SqlQueryContext ctx, EntityFilter entityFilter, List entityFieldsFilters) { String permissionQuery = this.buildPermissionQuery(ctx, entityFilter); String entityFilterQuery = this.buildEntityFilterQuery(ctx, entityFilter); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, entityFieldsFilters, entityFilter.getType()); @@ -538,7 +539,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return result; } - private String buildPermissionQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildPermissionQuery(SqlQueryContext ctx, EntityFilter entityFilter) { if (ctx.isIgnorePermissionCheck()) { return "1=1"; } @@ -575,7 +576,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String defaultPermissionQuery(QueryContext ctx) { + private String defaultPermissionQuery(SqlQueryContext ctx) { ctx.addUuidParameter("permissions_tenant_id", ctx.getTenantId().getId()); if (ctx.getCustomerId() != null && !ctx.getCustomerId().isNullUid()) { ctx.addUuidParameter("permissions_customer_id", ctx.getCustomerId().getId()); @@ -593,7 +594,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String buildEntityFilterQuery(QueryContext ctx, EntityFilter entityFilter) { + private String buildEntityFilterQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case SINGLE_ENTITY: return this.singleEntityQuery(ctx, (SingleEntityFilter) entityFilter); @@ -619,7 +620,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String addEntityTableQuery(QueryContext ctx, EntityFilter entityFilter) { + private String addEntityTableQuery(SqlQueryContext ctx, EntityFilter entityFilter) { switch (entityFilter.getType()) { case RELATIONS_QUERY: return relationQuery(ctx, (RelationsQueryFilter) entityFilter); @@ -640,7 +641,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String entitySearchQuery(QueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { + private String entitySearchQuery(SqlQueryContext ctx, EntitySearchQueryFilter entityFilter, EntityType entityType, List types) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, additional_info " @@ -680,7 +681,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return query; } - private String relationQuery(QueryContext ctx, RelationsQueryFilter entityFilter) { + private String relationQuery(SqlQueryContext ctx, RelationsQueryFilter entityFilter) { EntityId rootId = entityFilter.getRootEntity(); String lvlFilter = getLvlFilter(entityFilter.getMaxLevel()); String selectFields = SELECT_TENANT_ID + ", " + SELECT_CUSTOMER_ID @@ -763,7 +764,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return "( " + selectFields + from + ")"; } - private String buildEtfCondition(QueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { + private String buildEtfCondition(SqlQueryContext ctx, RelationEntityTypeFilter etf, EntitySearchDirection direction, int entityTypeFilterIdx) { StringBuilder whereFilter = new StringBuilder(); String relationType = etf.getRelationType(); List entityTypes = etf.getEntityTypes(); @@ -812,7 +813,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return from; } - private String buildAliasWhereQuery(QueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { + private String buildAliasWhereQuery(SqlQueryContext ctx, EntityFilter entityFilter, List selectionMapping, String searchText) { List aliasFiltersMapping = selectionMapping.stream().filter(mapping -> !mapping.isLatest() && mapping.getEntityKeyColumn() == null) .collect(Collectors.toList()); String entityFieldsQuery = EntityKeyMapping.buildQuery(ctx, aliasFiltersMapping, entityFilter.getType()); @@ -822,12 +823,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { result += " where (" + entityFieldsQuery + ")"; } if (!searchTextQuery.isEmpty()) { - result += (result.isEmpty() ? " where ": " and ") + "(" + searchTextQuery + ") "; + result += (result.isEmpty() ? " where " : " and ") + "(" + searchTextQuery + ") "; } return result; } - private String buildTextSearchQuery(QueryContext ctx, List selectionMapping, String searchText) { + private String buildTextSearchQuery(SqlQueryContext ctx, List selectionMapping, String searchText) { if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) { String sqlSearchText = "%" + searchText + "%"; ctx.addStringParameter("lowerSearchTextParam", sqlSearchText); @@ -844,17 +845,17 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } } - private String singleEntityQuery(QueryContext ctx, SingleEntityFilter filter) { + private String singleEntityQuery(SqlQueryContext ctx, SingleEntityFilter filter) { ctx.addUuidParameter("entity_filter_single_entity_id", filter.getSingleEntity().getId()); return "e.id=:entity_filter_single_entity_id"; } - private String entityListQuery(QueryContext ctx, EntityListFilter filter) { + private String entityListQuery(SqlQueryContext ctx, EntityListFilter filter) { ctx.addUuidListParameter("entity_filter_entity_ids", filter.getEntityList().stream().map(UUID::fromString).collect(Collectors.toList())); return "e.id in (:entity_filter_entity_ids)"; } - private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { + private String entityNameQuery(SqlQueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); String nameColumn = getNameColumn(filter.getEntityType()); if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { @@ -864,7 +865,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return String.format("e.%s ilike concat(:entity_filter_name_filter, '%%')", nameColumn); } - private String typeQuery(QueryContext ctx, EntityFilter filter) { + private String typeQuery(SqlQueryContext ctx, EntityFilter filter) { List types; String name; String nameColumn; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java index 643b4edeec..a6ab1bee7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java @@ -37,7 +37,7 @@ public class DefaultQueryLogComponent implements QueryLogComponent { private long logQueriesThreshold; @Override - public void logQuery(QueryContext ctx, String query, long duration) { + public void logQuery(SqlQueryContext ctx, String query, long duration) { if (logSqlQueries && duration > logQueriesThreshold) { String sqlToUse = substituteParametersInSqlString(query, ctx); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java new file mode 100644 index 0000000000..91934293a6 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.query; + +import com.google.common.util.concurrent.ListenableFuture; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsMsg; +import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsRequest; +import org.thingsboard.server.common.data.edqs.query.EdqsResponse; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.edqs.EdqsService; + +@Service +@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "false", matchIfMissing = true) +public class DummyEdqsService implements EdqsService { + + @Override + public ListenableFuture processRequest(TenantId tenantId, CustomerId customerId, EdqsRequest request) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isApiEnabled() { + return false; + } + + @Override + public void onUpdate(TenantId tenantId, EntityId entityId, Object entity) {} + + @Override + public void onUpdate(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void onDelete(TenantId tenantId, EntityId entityId) {} + + @Override + public void onDelete(TenantId tenantId, ObjectType objectType, EdqsObject object) {} + + @Override + public void processSystemRequest(ToCoreEdqsRequest request) {} + + @Override + public void processSystemMsg(ToCoreEdqsMsg request) {} + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 599ac3d918..b02c208dae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -103,7 +103,7 @@ public class EntityKeyMapping { public static final List labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, ADDITIONAL_INFO); public static final List contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO); - public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); + public static final Set apiUsageStateEntityFields = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME)); public static final Set commonEntityFieldsSet = new HashSet<>(commonEntityFields); public static final Set relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO, RELATED_PARENT_ID)); @@ -265,7 +265,7 @@ public class EntityKeyMapping { return alias; } - public Stream toQueries(QueryContext ctx, EntityFilterType filterType) { + public Stream toQueries(SqlQueryContext ctx, EntityFilterType filterType) { if (hasFilter()) { String keyAlias = (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD) && getEntityKeyColumn() != null) ? "e" : alias; return keyFilters.stream().map(keyFilter -> @@ -275,7 +275,7 @@ public class EntityKeyMapping { } } - public String toLatestJoin(QueryContext ctx, EntityFilter entityFilter, EntityType entityType) { + public String toLatestJoin(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType) { String entityTypeStr; if (entityFilter.getType().equals(EntityFilterType.RELATIONS_QUERY)) { entityTypeStr = "entities.entity_type"; @@ -303,9 +303,9 @@ public class EntityKeyMapping { if (entityKey.getType().equals(EntityKeyType.CLIENT_ATTRIBUTE)) { scope = AttributeScope.CLIENT_SCOPE.getId(); } else if (entityKey.getType().equals(EntityKeyType.SHARED_ATTRIBUTE)) { - scope = AttributeScope.SHARED_SCOPE.getId();; + scope = AttributeScope.SHARED_SCOPE.getId(); ; } else { - scope = AttributeScope.SERVER_SCOPE.getId();; + scope = AttributeScope.SERVER_SCOPE.getId(); ; } query = String.format("%s AND %s.attribute_type=%s %s", query, alias, scope, filterQuery); } else { @@ -318,7 +318,7 @@ public class EntityKeyMapping { } } - private boolean hasFilterValues(QueryContext ctx) { + private boolean hasFilterValues(SqlQueryContext ctx) { return Arrays.stream(ctx.getParameterNames()).anyMatch(parameterName -> { return !parameterName.equals(getKeyId()) && parameterName.startsWith(alias); }); @@ -333,14 +333,14 @@ public class EntityKeyMapping { Collectors.joining(", ")); } - public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { + public static String buildLatestJoins(SqlQueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { return latestMappings.stream() .filter(mapping -> !countQuery || mapping.hasFilter()) .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)) .collect(Collectors.joining(" ")); } - public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { + public static String buildQuery(SqlQueryContext ctx, List mappings, EntityFilterType filterType) { return mappings.stream() .flatMap(mapping -> mapping.toQueries(ctx, filterType)) .filter(StringUtils::isNotEmpty) @@ -510,12 +510,12 @@ public class EntityKeyMapping { return getValueAlias() + "_so_num"; } - private String buildKeyQuery(QueryContext ctx, String alias, KeyFilter keyFilter, + private String buildKeyQuery(SqlQueryContext ctx, String alias, KeyFilter keyFilter, EntityFilterType filterType) { return this.buildPredicateQuery(ctx, alias, keyFilter.getKey(), keyFilter.getPredicate(), filterType); } - private String buildPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (predicate.getType().equals(FilterPredicateType.COMPLEX)) { return this.buildComplexPredicateQuery(ctx, alias, key, (ComplexFilterPredicate) predicate, filterType); @@ -524,7 +524,7 @@ public class EntityKeyMapping { } } - private String buildComplexPredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildComplexPredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, ComplexFilterPredicate predicate, EntityFilterType filterType) { String result = predicate.getPredicates().stream() .map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate, filterType)) @@ -536,7 +536,7 @@ public class EntityKeyMapping { return result; } - private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key, + private String buildSimplePredicateQuery(SqlQueryContext ctx, String alias, EntityKey key, KeyFilterPredicate predicate, EntityFilterType filterType) { if (key.getType().equals(EntityKeyType.ENTITY_FIELD)) { String field = (getEntityKeyColumn() != null) ? alias + "." + getEntityKeyColumn() : alias; @@ -571,7 +571,7 @@ public class EntityKeyMapping { } } - private String buildStringPredicateQuery(QueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { + private String buildStringPredicateQuery(SqlQueryContext ctx, String field, StringFilterPredicate stringFilterPredicate) { String operationField = field; String paramName = getNextParameterName(field); String value = stringFilterPredicate.getValue().getValue(); @@ -624,7 +624,7 @@ public class EntityKeyMapping { return String.format("((%s is not null and %s)", field, stringOperationQuery); } - private String buildNumericPredicateQuery(QueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { + private String buildNumericPredicateQuery(SqlQueryContext ctx, String field, NumericFilterPredicate numericFilterPredicate) { String paramName = getNextParameterName(field); ctx.addDoubleParameter(paramName, numericFilterPredicate.getValue().getValue()); String numericOperationQuery = ""; @@ -651,7 +651,7 @@ public class EntityKeyMapping { return String.format("(%s is not null and %s)", field, numericOperationQuery); } - private String buildBooleanPredicateQuery(QueryContext ctx, String field, + private String buildBooleanPredicateQuery(SqlQueryContext ctx, String field, BooleanFilterPredicate booleanFilterPredicate) { String paramName = getNextParameterName(field); ctx.addBooleanParameter(paramName, booleanFilterPredicate.getValue().getValue()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java index f455e2d591..cbb3d4141d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java @@ -17,5 +17,5 @@ package org.thingsboard.server.dao.sql.query; public interface QueryLogComponent { - void logQuery(QueryContext ctx, String query, long duration); + void logQuery(SqlQueryContext ctx, String query, long duration); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java similarity index 94% rename from dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java rename to dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java index 6869703bce..42ac8280ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryContext.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/SqlQueryContext.java @@ -21,6 +21,7 @@ import org.springframework.jdbc.core.namedparam.SqlParameterSource; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; import java.sql.Types; import java.util.HashMap; @@ -29,14 +30,14 @@ import java.util.Map; import java.util.UUID; @Slf4j -public class QueryContext implements SqlParameterSource { +public class SqlQueryContext implements SqlParameterSource { private static final UUIDJdbcType UUID_TYPE = UUIDJdbcType.INSTANCE; - private final QuerySecurityContext securityCtx; + private final QueryContext securityCtx; private final StringBuilder query; private final Map params; - public QueryContext(QuerySecurityContext securityCtx) { + public SqlQueryContext(QueryContext securityCtx) { this.securityCtx = securityCtx; query = new StringBuilder(); params = new HashMap<>(); @@ -48,7 +49,7 @@ public class QueryContext implements SqlParameterSource { if (oldParam != null && oldParam.value != null && !oldParam.value.equals(newParam.value)) { throw new RuntimeException("Parameter with name: " + name + " was already registered!"); } - if(value == null){ + if (value == null) { log.warn("[{}][{}][{}] Trying to set null value", getTenantId(), getCustomerId(), name); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index e4563b7d35..c84392842a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -22,11 +22,13 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.QueueEntity; import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaQueueDao extends JpaAbstractDao implements QueueDao { +public class JpaQueueDao extends JpaAbstractDao implements QueueDao, TenantEntityDao { @Autowired private QueueRepository queueRepository; @@ -87,9 +89,19 @@ public class JpaQueueDao extends JpaAbstractDao implements Q .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findQueuesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE; } + @Override + public ObjectType getType() { + return ObjectType.QUEUE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index f680a42dc1..b78bbb8f37 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.common.data.id.QueueStatsId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -74,6 +75,11 @@ public class JpaQueueStatsDao extends JpaAbstractDao findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(queueStatsRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.QUEUE_STATS; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java index 585f010469..594e80e42a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.QueueStatsFields; import org.thingsboard.server.dao.model.sql.QueueStatsEntity; import java.util.List; @@ -45,4 +46,8 @@ public interface QueueStatsRepository extends JpaRepository findByTenantIdAndIdIn(UUID tenantId, List queueStatsIds); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.QueueStatsFields(q.id, q.createdTime," + + "q.tenantId, q.queueName, q.serviceId) FROM QueueStatsEntity q") + Page findAllFields(Pageable pageable); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 0e726f8917..1df7115518 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -24,6 +24,8 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -127,6 +129,11 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.toPageData(relationRepository.findAll(DaoUtil.toPageable(pageLink))); + } + @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 0f56a413df..d945f1f640 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; @@ -84,4 +85,6 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); + @Query("SELECT e FROM RelationEntity e ORDER BY e.fromId, e.fromType, e.toId, e.toType, e.relationType, e.relationTypeGroup") + Page findAll(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index f8927aaa81..ca0045b153 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.ResourceSubType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.TbResourceEntity; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -38,7 +40,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao { +public class JpaTbResourceDao extends JpaAbstractDao implements TbResourceDao, TenantEntityDao { private final TbResourceRepository resourceRepository; @@ -145,4 +147,9 @@ public class JpaTbResourceDao extends JpaAbstractDao implements RpcDao { +public class JpaRpcDao extends JpaAbstractDao implements RpcDao, TenantEntityDao { private final RpcRepository rpcRepository; @@ -74,9 +76,19 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findAllRpcByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RPC; } + @Override + public ObjectType getType() { + return ObjectType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index eaf151f256..588be25ba9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -20,6 +20,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -133,9 +135,24 @@ public class JpaRuleChainDao extends JpaAbstractDao return findRootRuleChainByTenantIdAndType(tenantId, RuleChainType.CORE); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(ruleChainRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; } + @Override + public ObjectType getType() { + return ObjectType.RULE_CHAIN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index c267eab4a8..307f15c7f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -40,7 +42,7 @@ import java.util.stream.Collectors; @Slf4j @Component @SqlDao -public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao { +public class JpaRuleNodeDao extends JpaAbstractDao implements RuleNodeDao, TenantEntityDao { @Autowired private RuleNodeRepository ruleNodeRepository; @@ -106,9 +108,19 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(ruleNodeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.RULE_NODE; } + @Override + public ObjectType getType() { + return ObjectType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 305d5993d4..6713e4b645 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; @@ -70,4 +71,7 @@ public interface RuleChainRepository extends JpaRepository findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index 264ea88326..bf19d61a9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -63,4 +63,7 @@ public interface RuleNodeRepository extends JpaRepository @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") void deleteByIdIn(@Param("ids") List ids); + @Query("SELECT n FROM RuleNodeEntity n WHERE n.ruleChainId IN (SELECT rc.id FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index b390bfffd8..78cf891870 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.settings; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; @@ -33,4 +35,6 @@ public interface AdminSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 9aa348876e..9e0f6a70cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,7 +21,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -32,7 +37,7 @@ import java.util.UUID; @Component @SqlDao @Slf4j -public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { +public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao, TenantEntityDao { @Autowired private AdminSettingsRepository adminSettingsRepository; @@ -68,4 +73,14 @@ public class JpaAdminSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(adminSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.ADMIN_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index cea414a42f..ff014d012f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; @@ -94,4 +95,15 @@ public class JpaTenantDao extends JpaAbstractDao implement .map(TenantId::fromUUID) .collect(Collectors.toList()); } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(tenantRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java index 372c5bc9b8..4668c44b61 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantProfileDao.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -87,6 +88,11 @@ public class JpaTenantProfileDao extends JpaAbstractDao findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(tenantProfileRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.TENANT_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java index a9cbf3d591..b7a57d0942 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantProfileRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityInfo; +import org.thingsboard.server.common.data.edqs.fields.TenantProfileFields; import org.thingsboard.server.dao.model.sql.TenantProfileEntity; import java.util.List; @@ -55,4 +56,8 @@ public interface TenantProfileRepository extends JpaRepository findByIdIn(List ids); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantProfileFields(t.id, t.createdTime, t.name," + + "t.isDefault) FROM TenantProfileEntity t") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 795404399e..86dbeaff80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.TenantFields; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.model.sql.TenantInfoEntity; @@ -53,4 +54,8 @@ public interface TenantRepository extends JpaRepository { @Query("SELECT t.id FROM TenantEntity t where t.tenantProfileId = :tenantProfileId") List findTenantIdsByTenantProfileId(@Param("tenantProfileId") UUID tenantProfileId); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.TenantFields(t.id, t.createdTime, t.title, t.version," + + "t.additionalInfo, t.country, t.state, t.city, t.address, t.address2, t.zip, t.phone, t.email, t.region) FROM TenantEntity t") + Page findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 88873e8671..509796879c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -15,11 +15,14 @@ */ package org.thingsboard.server.dao.sql.usagerecord; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import java.util.UUID; @@ -35,6 +38,8 @@ public interface ApiUsageStateRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying @Query("DELETE FROM ApiUsageStateEntity ur WHERE ur.tenantId = :tenantId") @@ -44,4 +49,9 @@ public interface ApiUsageStateRepository extends JpaRepository findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 63c0d9d3d1..85d9d18839 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -19,8 +19,12 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.ApiUsageStateFields; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiUsageStateEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -72,9 +76,24 @@ public class JpaApiUsageStateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiUsageStateRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(apiUsageStateRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; } + @Override + public ObjectType getType() { + return ObjectType.API_USAGE_STATE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index 316e34de87..171d10b0c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -18,9 +18,15 @@ package org.thingsboard.server.dao.sql.user; import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserAuthSettings; +import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserAuthSettingsDao; @@ -31,7 +37,7 @@ import java.util.UUID; @Component @RequiredArgsConstructor @SqlDao -public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao { +public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserAuthSettingsDao, TenantEntityDao { private final UserAuthSettingsRepository repository; @@ -45,6 +51,18 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + PageData data = DaoUtil.toPageData(repository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + data.getData().forEach(settings -> { + AccountTwoFaSettings twoFaSettings = settings.getTwoFaSettings(); + if (twoFaSettings != null && twoFaSettings.getConfigs() != null) { + twoFaSettings.getConfigs().values().forEach(config -> config.setSerializeHiddenFields(true)); + } + }); + return data; + } + @Override protected Class getEntityClass() { return UserAuthSettingsEntity.class; @@ -55,4 +73,9 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao implements UserCredentialsDao { +public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao, TenantEntityDao { @Autowired private UserCredentialsRepository userCredentialsRepository; @@ -84,4 +88,14 @@ public class JpaUserCredentialsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userCredentialsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.USER_CREDENTIALS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 7749dbff29..55d74634d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; @@ -139,9 +141,24 @@ public class JpaUserDao extends JpaAbstractDao implements User return userRepository.countByTenantId(tenantId.getId()); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(userRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.USER; } + @Override + public ObjectType getType() { + return ObjectType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index 671304256e..b41bfa7728 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -18,14 +18,17 @@ package org.thingsboard.server.dao.sql.user; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.common.data.settings.UserSettingsType; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.user.UserSettingsDao; import org.thingsboard.server.dao.util.SqlDao; @@ -34,7 +37,7 @@ import java.util.List; @Slf4j @Component @SqlDao -public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService implements UserSettingsDao { +public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao { @Autowired private UserSettingsRepository userSettingsRepository; @@ -66,4 +69,14 @@ public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService i return DaoUtil.convertDataList(userSettingsRepository.findByTypeAndPathExisting(type.name(), path)); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(userSettingsRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.USER_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index cea5d428dd..3c9d69f085 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -35,4 +37,7 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 4aca40647c..682219ed3a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; @@ -52,4 +55,7 @@ public interface UserCredentialsRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 79b0a00308..e67330a1a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.UserFields; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; @@ -71,4 +72,9 @@ public interface UserRepository extends JpaRepository { Long countByTenantId(UUID tenantId); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.UserFields(u.id, u.createdTime, u.tenantId," + + "u.customerId, u.version, u.firstName, u.lastName, u.email, u.phone, u.additionalInfo) " + + "FROM UserEntity u") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index 436062ef4c..9a0add31ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.user; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -36,4 +38,7 @@ public interface UserSettingsRepository extends JpaRepository findByTypeAndPathExisting(@Param("type") String type, @Param("path") String[] path); + @Query("SELECT s FROM UserSettingsEntity s WHERE s.userId IN (SELECT u.id FROM UserEntity u WHERE u.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 6220d9b717..ed0c5155b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -19,6 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.UserFields; +import org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.page.PageData; @@ -30,6 +33,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeFilter; import org.thingsboard.server.common.data.widget.WidgetTypeInfo; import org.thingsboard.server.common.data.widget.WidgetsBundleWidget; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetTypeDetailsEntity; import org.thingsboard.server.dao.model.sql.WidgetTypeInfoEntity; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; @@ -53,7 +57,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { +public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao, TenantEntityDao { @Autowired private WidgetTypeRepository widgetTypeRepository; @@ -255,10 +259,24 @@ public class JpaWidgetTypeDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(widgetTypeRepository.findAllFields(DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.WIDGET_TYPE; } + @Override + public ObjectType getType() { + return ObjectType.WIDGET_TYPE; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 290e949c38..5c9b4127f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -19,6 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; @@ -26,6 +28,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.data.widget.WidgetsBundleFilter; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; @@ -44,7 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Component @SqlDao -public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao { +public class JpaWidgetsBundleDao extends JpaAbstractDao implements WidgetsBundleDao, TenantEntityDao { @Autowired private WidgetsBundleRepository widgetsBundleRepository; @@ -155,7 +158,17 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findByImageLink(String imageUrl, int limit) { - return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + return DaoUtil.convertDataList(widgetsBundleRepository.findByImageUrl(imageUrl, limit)); + } + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return findByTenantId(tenantId.getId(), pageLink); + } + + @Override + public PageData findAllFields(PageLink pageLink) { + return DaoUtil.pageToPageData(widgetsBundleRepository.findAllFields(DaoUtil.toPageable(pageLink))); } @Override @@ -163,4 +176,9 @@ public class JpaWidgetsBundleDao extends JpaAbstractDao findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetTypeFields(w.id, w.createdTime, w.tenantId," + + "w.name, w.version) FROM WidgetTypeEntity w") + Page findAllFields(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 8de78986ae..97b2f52d63 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; @@ -139,4 +140,8 @@ public interface WidgetsBundleRepository extends JpaRepository findByImageUrl(@Param("imageLink") String imageLink, @Param("lmt") int lmt); + + @Query("SELECT new org.thingsboard.server.common.data.edqs.fields.WidgetsBundleFields(w.id, w.createdTime, w.tenantId," + + "w.alias, w.version) FROM WidgetsBundleEntity w") + Page findAllFields(Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java index 89533106f3..3a024e1687 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java @@ -15,7 +15,11 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetEntity; @@ -26,4 +30,7 @@ public interface WidgetsBundleWidgetRepository extends JpaRepository findAllByWidgetsBundleId(UUID widgetsBundleId); + @Query("SELECT w FROM WidgetsBundleWidgetEntity w WHERE w.widgetsBundleId IN (SELECT b.id FROM WidgetsBundleEntity b WHERE b.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 18364c6b1b..b134b6308d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -39,7 +39,6 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @@ -119,4 +118,5 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries protected int getDataPointDays(TsKvEntry tsKvEntry, long ttl) { return tsKvEntry.getDataPoints() * Math.max(1, (int) (ttl / SECONDS_IN_DAY)); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 0be078a89f..0e9cebd150 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -33,14 +33,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import org.thingsboard.server.dao.timeseries.TsLatestCacheKey; import org.thingsboard.server.dao.util.SqlTsLatestAnyDaoCachedRedis; import java.util.List; +import java.util.Map; import java.util.Optional; @Slf4j @@ -167,4 +171,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return null; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index 44859c26e7..fd1b180eef 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,6 +24,7 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,8 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -185,6 +188,11 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return DaoUtil.pageToPageData(tsKvLatestRepository.findAll(DaoUtil.toPageable(pageLink, "entityId", "key"))); + } + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); return Futures.transformAsync(future, entryList -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java index b01d1c4ea0..7194ae957c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/JpaKeyDictionaryDao.java @@ -22,6 +22,9 @@ import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -92,4 +95,9 @@ public class JpaKeyDictionaryDao extends JpaAbstractDaoListeningExecutorService return byKeyId.map(KeyDictionaryEntry::getKey).orElse(null); } + @Override + public PageData findAll(PageLink pageLink) { + return DaoUtil.pageToPageData(keyDictionaryRepository.findAll(DaoUtil.toPageable(pageLink))); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java index 17e24ea5e5..0667f4315e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/KeyDictionaryRepository.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.dao.sqlts.dictionary; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryCompositeKey; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; @@ -25,5 +28,7 @@ public interface KeyDictionaryRepository extends JpaRepository findByKeyId(int keyId); + @Query("SELECT e FROM KeyDictionaryEntry e ORDER BY e.keyId ASC") + Page findAll(Pageable pageable); } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 38bfde0acf..833ffd185e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sqlts.latest; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; @@ -41,4 +43,7 @@ public interface TsKvLatestRepository extends JpaRepository findAllKeysByEntityIds(@Param("entityIds") List entityIds); + @Query("SELECT e FROM TsKvLatestEntity e ORDER BY e.entityId ASC, e.key ASC") + Page findAll(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java index 5fb5ff28c4..40ec208f08 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantDao.java @@ -37,10 +37,10 @@ public interface TenantDao extends Dao { * @return saved tenant object */ Tenant save(TenantId tenantId, Tenant tenant); - + /** * Find tenants by page link. - * + * * @param pageLink the page link * @return the list of tenant objects */ @@ -51,4 +51,5 @@ public interface TenantDao extends Dao { PageData findTenantsIds(PageLink pageLink); List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 756b73d88b..cb731a782f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -26,6 +26,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; @@ -38,6 +40,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.Validator; @@ -54,7 +57,6 @@ import static org.thingsboard.server.common.data.StringUtils.isBlank; /** * @author Andrew Shvayka */ -@SuppressWarnings("UnstableApiUsage") @Service @Slf4j public class BaseTimeseriesService implements TimeseriesService { @@ -89,6 +91,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private EntityViewService entityViewService; + @Autowired + private EdqsService edqsService; + @Override public ListenableFuture> findAllByQueries(TenantId tenantId, EntityId entityId, List queries) { validate(entityId); @@ -190,14 +195,21 @@ public class BaseTimeseriesService implements TimeseriesService { public ListenableFuture> saveLatest(TenantId tenantId, EntityId entityId, List tsKvEntries) { List> futures = new ArrayList<>(tsKvEntries.size()); for (TsKvEntry tsKvEntry : tsKvEntries) { - futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(doSaveLatest(tenantId, entityId, tsKvEntry)); } return Futures.allAsList(futures); } private void saveAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { doSaveAndRegisterFuturesFor(tenantId, futures, entityId, tsKvEntry, ttl); - futures.add(Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); + futures.add(Futures.transform(doSaveLatest(tenantId, entityId, tsKvEntry), v -> 0, MoreExecutors.directExecutor())); + } + + private ListenableFuture doSaveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return Futures.transform(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry), version -> { + edqsService.onUpdate(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, version)); + return version; + }, MoreExecutors.directExecutor()); } private void saveWithoutLatestAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { @@ -248,7 +260,7 @@ public class BaseTimeseriesService implements TimeseriesService { List> futures = new ArrayList<>(keys.size()); for (String key : keys) { DeleteTsKvQuery query = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false); - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } return Futures.allAsList(futures); } @@ -269,10 +281,20 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(Futures.transform(timeseriesDao.remove(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); if (query.getDeleteLatest()) { - futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); + futures.add(doRemove(tenantId, entityId, query)); } } + private ListenableFuture doRemove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return Futures.transform(timeseriesLatestDao.removeLatest(tenantId, entityId, query), result -> { + if (result.isRemoved()) { + Long version = result.getVersion(); + edqsService.onDelete(tenantId, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, query.getKey(), version)); + } + return result; + }, MoreExecutors.directExecutor()); + } + private static void validate(EntityId entityId) { Validator.validateEntityId(entityId, id -> "Incorrect entityId " + id); } @@ -302,4 +324,5 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Incorrect DeleteTsKvQuery. Key can't be empty"); } } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 01c91d4801..99bf57a1db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,13 +36,17 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -99,6 +103,11 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override + public PageData findAllLatest(PageLink pageLink) { + return null; + } + @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 9f62fd033a..1372f2f5d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,8 +22,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; +import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -49,4 +53,6 @@ public interface TimeseriesLatestDao { List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + + PageData findAllLatest(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java index ce71b733e5..d0da2a3943 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateDao.java @@ -19,10 +19,11 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface ApiUsageStateDao extends Dao { +public interface ApiUsageStateDao extends Dao, TenantEntityDao { /** * Save or update usage record object @@ -50,4 +51,5 @@ public interface ApiUsageStateDao extends Dao { void deleteApiUsageStateByTenantId(TenantId tenantId); void deleteApiUsageStateByEntityId(EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index d901ae950e..9512120a3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -113,6 +113,14 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A ApiUsageState saved = apiUsageStateDao.save(apiUsageState.getTenantId(), apiUsageState); + eventPublisher.publishEvent(SaveEntityEvent.builder() + .tenantId(saved.getTenantId()) + .entityId(saved.getId()) + .entity(saved) + .created(true) + .broadcastEvent(false) + .build()); + List apiUsageStates = new ArrayList<>(); apiUsageStates.add(new BasicTsKvEntry(saved.getCreatedTime(), new StringDataEntry(ApiFeature.TRANSPORT.getApiStateKey(), ApiUsageStateValue.ENABLED.name()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index 42add1bbe2..78b9455586 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 7d8018cdd8..f8a0b43741 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -77,7 +77,6 @@ import java.util.UUID; import static org.junit.Assert.assertNotNull; - @RunWith(SpringRunner.class) @ContextConfiguration(classes = AbstractServiceTest.class, loader = AnnotationConfigContextLoader.class) @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) @@ -131,7 +130,7 @@ public abstract class AbstractServiceTest { } public JsonNode readFromResource(String resourceName) throws IOException { - try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)){ + try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(resourceName)) { return JacksonUtil.fromBytes(Objects.requireNonNull(is).readAllBytes()); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java index 30110a2741..e06d83682a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -21,16 +21,31 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; +import static org.thingsboard.server.common.data.ObjectType.EVENT; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_CLIENT; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_DOMAIN; +import static org.thingsboard.server.common.data.ObjectType.OAUTH2_MOBILE; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TENANT_PROFILE; @Slf4j @DaoSqlTest @@ -88,4 +103,20 @@ public class EntityDaoRegistryTest extends AbstractServiceTest { } } + @Test + public void givenAllTenantEntityDaos_whenFindAllByTenantId_thenOk() { + Set ignored = EnumSet.of(TENANT, TENANT_PROFILE, RELATION, EVENT, ATTRIBUTE_KV, LATEST_TS_KV, AUDIT_LOG, + OAUTH2_CLIENT, OAUTH2_DOMAIN, OAUTH2_MOBILE); + for (ObjectType type : ObjectType.values()) { + if (ignored.contains(type)) { + continue; + } + + TenantEntityDao dao = assertDoesNotThrow(() -> entityDaoRegistry.getTenantEntityDao(type)); + assertDoesNotThrow(() -> { + dao.findAllByTenantId(tenantId, new PageLink(100)); + }); + } + } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java index 2a0210f8c8..b880871bd6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponentTest.java @@ -29,6 +29,8 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit4.SpringRunner; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.permission.QueryContext; +import org.thingsboard.server.common.data.permission.QueryContext; import java.util.List; import java.util.UUID; @@ -48,7 +50,7 @@ import static org.mockito.Mockito.times; public class DefaultQueryLogComponentTest { private TenantId tenantId; - private QueryContext ctx; + private SqlQueryContext ctx; @SpyBean private DefaultQueryLogComponent queryLog; @@ -56,7 +58,7 @@ public class DefaultQueryLogComponentTest { @Before public void setUp() { tenantId = new TenantId(UUID.fromString("97275c1c-9cf2-4d25-a68d-933031158f84")); - ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); + ctx = new SqlQueryContext(new QueryContext(tenantId, null, EntityType.ALARM)); } @Test diff --git a/edqs/pom.xml b/edqs/pom.xml new file mode 100644 index 0000000000..bf83c75b32 --- /dev/null +++ b/edqs/pom.xml @@ -0,0 +1,260 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0PE-SNAPSHOT + thingsboard + + edqs + jar + + ThingsBoard Entity Data Query Service Application + https://thingsboard.io + ThingsBoard Professional Edition: IoT Platform - Device management, data collection, processing and visualization + + + + UTF-8 + ${basedir}/.. + java + false + process-resources + package + edqs + ${project.build.directory}/windows + true + ThingsBoard Entity Data Query Service + org.thingsboard.server.edqs.ThingsboardEdqsApplication + + + + + org.thingsboard.common + edqs + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.apache.curator + curator-recipes + + + com.google.protobuf + protobuf-java + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + + com.sun.winsw + winsw + bin + exe + provided + + + org.thingsboard + tools + test + + + org.springframework.security + spring-security-test + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.dbunit + dbunit + test + + + com.github.springtestdbunit + spring-test-dbunit + test + + + org.testcontainers + cassandra + test + + + org.testcontainers + postgresql + test + + + org.testcontainers + jdbc + test + + + org.java-websocket + Java-WebSocket + test + + + org.eclipse.milo + sdk-server + test + + + org.assertj + assertj-core + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + true + + edqs.yml + + + + ${project.basedir}/src/main/resources + false + + edqs.yml + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + thingsboard + + + **/nosql/*Test.java + + + **/*Test.java + **/*TestSuite.java + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-winsw-service + package + + + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + org.codehaus.mojo + build-helper-maven-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + + true + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java new file mode 100644 index 0000000000..ea8af83310 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyQueueRoutingInfoService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class DummyQueueRoutingInfoService implements QueueRoutingInfoService { + + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java new file mode 100644 index 0000000000..5037652e5c --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/DummyTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class DummyTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + return null; + } + +} diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java new file mode 100644 index 0000000000..1157f74de1 --- /dev/null +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.thingsboard.server.edqs.repo.EdqRepository; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment"}) +@Slf4j +public class ThingsboardEdqsApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "edqs"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardEdqsApplication.class, updateArguments(args)); + } + + // @Bean + public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { + return args -> { +// long startTs = System.currentTimeMillis(); +// var loader = new TenantRepoLoader(new TenantRepo(TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")))); +// loader.load(); +// log.info("Loaded all in {} ms", System.currentTimeMillis() - startTs); + + + +// log.info("Compressed {} strings/json, Before: {}, After: {}", +// CompressedStringDataPoint.cnt.get(), +// CompressedStringDataPoint.uncompressedLength.get(), +// CompressedStringDataPoint.compressedLength.get()); +// +// log.info("Deduplicated {} short and {} long strings", +// TbStringPool.size(), TbBytePool.size()); +// +// var tenantId = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); +// var customerId = new CustomerId(UUID.fromString("fcbf2f50-d0d9-11ea-bea3-177755191a6e")); +// System.gc(); +// +// while (true) { +// EntityTypeFilter filter = new EntityTypeFilter(); +// filter.setEntityType(EntityType.DEVICE); +// var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); +// +// var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); +// var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); +// KeyFilter nameFilter = new KeyFilter(); +// nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); +// var predicate = new StringFilterPredicate(); +// predicate.setIgnoreCase(false); +// predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); +// predicate.setValue(new FilterPredicateValue<>("LoRa-")); +// nameFilter.setPredicate(predicate); +// nameFilter.setValueType(EntityKeyValueType.STRING); +// +// EntityDataQuery edq = new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); +// var result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); +// log.info("Device count: {}", result.getTotalElements()); +// log.info("First: {}", result.getData().get(0).getEntityId()); +// log.info("Last: {}", result.getData().get(19).getEntityId()); +// +// pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.ASC)); +// result = edqRepository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, edq, false); +// log.info("Device count: {}", result.getTotalElements()); +// log.info("First: {}", result.getData().get(0).getEntityId()); +// log.info("Last: {}", result.getData().get(19).getEntityId()); +// +// result.getData().forEach(data -> { +// System.err.println(data.getEntityId() + ":"); +// data.getLatest().forEach((type, values) -> { +// System.err.println(type); +// values.forEach((key, tsValue) -> { +// System.err.println(key + " = " + tsValue.getValue()); +// }); +// }); +// System.err.println(); +// }); +// Thread.sleep(5000); +// } + }; + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } + +} diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml new file mode 100644 index 0000000000..f5c83178c3 --- /dev/null +++ b/edqs/src/main/resources/edqs.yml @@ -0,0 +1,364 @@ +# +# Copyright © 2016-2024 ThingsBoard, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Application info parameters +app: + # Application version + version: "@project.version@" + +# Zookeeper connection parameters +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. + # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. + # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. + recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" + +# SQL DAO Configuration parameters +spring: + main: + web-application-type: "none" + +# Queue configuration parameters +queue: + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) + prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). + in_memory: + stats: + # For debug level + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + edqs: + enabled: "${TB_EDQS_ENABLED:true}" + mode: "${TB_EDQS_MODE:local}" + partitions: "${TB_EDQS_PARTITIONS:12}" + requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" + responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" + max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data + kafka: + # Kafka Bootstrap nodes in "host:port" format + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + ssl: + # Enable/Disable SSL Kafka communication + enabled: "${TB_KAFKA_SSL_ENABLED:false}" + # The location of the trust store file + truststore.location: "${TB_KAFKA_SSL_TRUSTSTORE_LOCATION:}" + # The password of trust store file if specified + truststore.password: "${TB_KAFKA_SSL_TRUSTSTORE_PASSWORD:}" + # The location of the key store file. This is optional for the client and can be used for two-way authentication for the client + keystore.location: "${TB_KAFKA_SSL_KEYSTORE_LOCATION:}" + # The store password for the key store file. This is optional for the client and only needed if ‘ssl.keystore.location’ is configured. Key store password is not supported for PEM format + keystore.password: "${TB_KAFKA_SSL_KEYSTORE_PASSWORD:}" + # The password of the private key in the key store file or the PEM key specified in ‘keystore.key’ + key.password: "${TB_KAFKA_SSL_KEY_PASSWORD:}" + # The number of acknowledgments the producer requires the leader to have received before considering a request complete. This controls the durability of records that are sent. The following settings are allowed:0, 1 and all + acks: "${TB_KAFKA_ACKS:all}" + # Number of retries. Resend any record whose send fails with a potentially transient error + retries: "${TB_KAFKA_RETRIES:1}" + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # Default batch size. This setting gives the upper bound of the batch size to be sent + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + # The maximum size of a request in bytes. This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + # The maximum number of unacknowledged requests the client will send on a single connection before blocking + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + # The total bytes of memory the producer can use to buffer records waiting to be sent to the server + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + # The multiple copies of data over the multiple brokers of Kafka + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + # The maximum delay between invocations of poll() method when using consumer group management. This places an upper bound on the amount of time that the consumer can be idle before fetching more records + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + # The maximum number of records returned in a single call of poll() method + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + # The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + # The maximum amount of data the server will return. Records are fetched in batches by the consumer + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none + # Enable/Disable using of Confluent Cloud + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + # The endpoint identification algorithm used by clients to validate server hostname. The default value is https + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + # The mechanism used to authenticate Schema Registry requests. SASL/PLAIN should only be used with TLS/SSL as a transport layer to ensure that clear passwords are not transmitted on the wire without encryption + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + # Using JAAS Configuration for specifying multiple SASL mechanisms on a broker + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + - key: max.poll.records + # Example of specific consumer properties value per topic + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + # Example of specific consumer properties value per topic for VC + - key: max.poll.interval.ms + # Example of specific consumer properties value per topic for VC + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + # Consumer properties for Housekeeper tasks topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}" + tb_housekeeper.reprocessing: + # Consumer properties for Housekeeper reprocessing topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" + other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" + other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + # value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + # Kafka properties for Rule Engine + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Core topics + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Transport Api topics + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Notifications topics + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for JS Executor topics + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + # Kafka properties for OTA updates topic + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Version Control topic + version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Integration Api topics + integration-api: "${TB_QUEUE_KAFKA_INTEGRATION_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time + housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions + edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions + edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + consumer-stats: + # Prints lag between consumer group offset and last messages offset in Kafka topics + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + # Statistics printing interval for Kafka's consumer-groups stats + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + # Time to wait for the stats-loading requests to Kafka to finish + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + transport_api: + # Topic used to consume api requests from transport microservices + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + # Topic used to produce api responses to transport microservices + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + # Maximum pending api requests from transport microservices to be handled by server + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout in milliseconds to handle api request from transport microservice by server + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + # Amount of threads used to invoke callbacks + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + # Amount of threads used for transport API requests + max_core_handler_threads: "${TB_QUEUE_TRANSPORT_MAX_CORE_HANDLER_THREADS:16}" + # Interval in milliseconds to poll api requests from transport microservices + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + # Interval in milliseconds to poll api response from transport microservices + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + # Default topic name of Kafka, RabbitMQ, etc. queue + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # Interval in milliseconds to poll messages by Core microservices + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + # Amount of partitions used by Core microservices + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + # Timeout for processing a message pack by Core microservices + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + # Enable/disable a separate consumer per partition for Core queue + consumer-per-partition: "${TB_QUEUE_CORE_CONSUMER_PER_PARTITION:true}" + ota: + # Default topic name for OTA updates + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + # The interval of processing the OTA updates for devices. Used to avoid any harm to the network due to many parallel OTA updates + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + # Stats topic name for queue Kafka, RabbitMQ, etc. + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + # Enable/disable statistics for Core microservices + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + # Statistics printing interval for Core microservices + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + # Topic name for Housekeeper tasks to be reprocessed + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + # Poll interval for topics related to Housekeeper + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + # Comma-separated list of task types that shouldn't be processed. Available task types: + # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled), + # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS + disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" + # Delay in milliseconds between tasks reprocessing + task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}" + # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" + stats: + # Enable/disable statistics for Housekeeper + enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" + # Statistics printing interval for Housekeeper + print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + + vc: + # Default topic name for Kafka, RabbitMQ, etc. + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + # Interval in milliseconds between polling of the messages if no new messages arrive + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + # Timeout before retrying all failed and timed-out messages from the processing pack + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" + # Timeout for a request to VC-executor (for a request for the version of the entity, for a commit charge, etc.) + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" + # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" + # JS max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS execution max request timeout + max_exec_requests_timeout: "${REMOTE_JS_MAX_EXEC_REQUEST_TIMEOUT:2000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + rule-engine: + # Deprecated. It will be removed in the nearest releases + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # Interval in milliseconds to poll messages by Rule Engine + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + # Timeout for processing a message pack of Rule Engine + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" + stats: + # Enable/disable statistics for Rule Engine + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + # Statistics printing interval for Rule Engine + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + # Max length of the error message that is printed by statistics + max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" + # After a queue is deleted (or the profile's isolation option was disabled), Rule Engine will continue reading related topics during this period before deleting the actual topics + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:15}" + # Size of the thread pool that handles such operations as partition changes, config updates, queue deletion + management-thread-pool-size: "${TB_QUEUE_RULE_ENGINE_MGMT_THREAD_POOL_SIZE:12}" + transport: + # For high-priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + # Interval in milliseconds to poll messages + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + integration: + # Name of hash function used for consistent hash ring in Cluster Mode. See architecture docs for more details. Valid values - murmur3_32, murmur3_128 or sha256 + partitions: "${TB_QUEUE_INTEGRATION_PARTITIONS:3}" + # Default notification topic name used by queue + notifications_topic: "${TB_QUEUE_INTEGRATION_NOTIFICATIONS_TOPIC:tb_ie.notifications}" + # Default downlink topic name used by queue + downlink_topic: "${TB_QUEUE_INTEGRATION_DOWNLINK_TOPIC:tb_ie.downlink}" + # Default uplink topic name used by queue + uplink_topic: "${TB_QUEUE_INTEGRATION_UPLINK_TOPIC:tb_ie.uplink}" + # Interval in milliseconds to poll messages by integrations + poll_interval: "${TB_QUEUE_INTEGRATION_POLL_INTERVAL_MS:25}" + # Timeout for processing a message pack by integrations + pack-processing-timeout: "${TB_QUEUE_INTEGRATION_PACK_PROCESSING_TIMEOUT_MS:10000}" + integration_api: + # Default Integration Api request topic name used by queue + requests_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_REQUEST_TOPIC:tb_ie.api.requests}" + # Default Integration Api response topic name used by queue + responses_topic: "${TB_QUEUE_INTEGRATION_EXECUTOR_API_RESPONSE_TOPIC:tb_ie.api.responses}" + # Maximum pending api requests from integration executor to be handled by server< + max_pending_requests: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout in milliseconds to handle api request from integration executor microservice by server + max_requests_timeout: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_REQUEST_TIMEOUT:10000}" + # Amount of threads used to invoke callbacks + max_callback_threads: "${TB_QUEUE_INTEGRATION_EXECUTOR_MAX_CALLBACK_THREADS:10}" + # Interval in milliseconds to poll api requests from integration executor microservices + request_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_REQUEST_POLL_INTERVAL_MS:25}" + # Interval in milliseconds to poll api response from integration executor microservices + response_poll_interval: "${TB_QUEUE_INTEGRATION_EXECUTOR_RESPONSE_POLL_INTERVAL_MS:25}" + +# General service parameters +service: + type: "${TB_SERVICE_TYPE:edqs}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + edqs: + label: "${TB_EDQS_LABEL:}" # services with the same label will share the list of partitions + +# Metrics parameters +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + system_info: + # Persist frequency of system info (CPU, memory usage, etc.) in seconds + persist_frequency: "${METRICS_SYSTEM_INFO_PERSIST_FREQUENCY_SECONDS:60}" + # TTL in days for system info timeseries + ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" + +# General management parameters +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}' + health: + elasticsearch: + # Enable the org.springframework.boot.actuate.elasticsearch.ElasticsearchRestClientHealthIndicator.doHealthCheck + enabled: "false" diff --git a/edqs/src/main/resources/logback.xml b/edqs/src/main/resources/logback.xml new file mode 100644 index 0000000000..09aa2b1ecb --- /dev/null +++ b/edqs/src/main/resources/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java new file mode 100644 index 0000000000..f6f3fcc60e --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -0,0 +1,296 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Before; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.test.context.TestExecutionListeners; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.support.DependencyInjectionTestExecutionListener; +import org.springframework.test.context.support.DirtiesContextTestExecutionListener; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.EdqsObject; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.SchedulerEventId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.scheduler.SchedulerEvent; +import org.thingsboard.server.edqs.processor.EdqsConverter; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@RunWith(SpringRunner.class) +@Configuration +@ComponentScan({"org.thingsboard.server.edqs.repo"}) +@EntityScan("org.thingsboard.server.edqs") +@TestPropertySource(locations = {"classpath:edq-test.properties"}) +@TestExecutionListeners({ + DependencyInjectionTestExecutionListener.class, + DirtiesContextTestExecutionListener.class}) +public abstract class AbstractEDQTest { + + @Autowired + protected InMemoryEdqRepository repository; + + protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); + + protected final UUID defaultAssetProfileId = UUID.randomUUID(); + protected final UUID defaultDeviceProfileId = UUID.randomUUID(); + + @Before + public final void before() { + AssetProfile ap = new AssetProfile(new AssetProfileId(defaultAssetProfileId)); + ap.setName("default"); + ap.setDefault(true); + addOrUpdate(EntityType.ASSET_PROFILE, ap); + + DeviceProfile dp = new DeviceProfile(new DeviceProfileId(defaultDeviceProfileId)); + dp.setName("default"); + dp.setDefault(true); + dp.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, dp); + + createCustomer(customerId.getId(), null, "Customer A"); + } + + @After + public final void after() { + repository.clear(); + } + + protected void createCustomer(UUID id, UUID parentCustomerId, String title) { + Customer entity = new Customer(); + entity.setId(new CustomerId(id)); + entity.setTitle(title); + entity.setOwnerId(parentCustomerId != null ? new CustomerId(parentCustomerId) : tenantId); + addOrUpdate(EntityType.CUSTOMER, entity); + } + + protected UUID createGroup(EntityType entityType, String groupName) { + return createGroup(null, entityType, groupName); + } + + protected UUID createGroup(UUID customerId, EntityType entityType, String groupName) { + EntityGroup eg = new EntityGroup(); + eg.setId(new EntityGroupId(UUID.randomUUID())); + eg.setType(entityType); + eg.setName(groupName); + eg.setOwnerId(customerId != null ? new CustomerId(customerId) : tenantId); + addOrUpdate(EntityType.ENTITY_GROUP, eg); + return eg.getId().getId(); + } + + protected UUID createDevice(String name) { + return createDevice(null, defaultDeviceProfileId, name); + } + + protected UUID createDevice(CustomerId customerId, String name) { + return createDevice(customerId.getId(), defaultDeviceProfileId, name); + } + + protected UUID createDevice(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Device entity = new Device(); + entity.setId(new DeviceId(entityId)); + if (profileId != null) { + entity.setDeviceProfileId(new DeviceProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createDashboard(String name) { + return createDashboard(null, name); + } + + protected UUID createDashboard(UUID customerId, String name) { + UUID entityId = UUID.randomUUID(); + Dashboard entity = new Dashboard(); + entity.setId(new DashboardId(entityId)); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setTitle(name); + addOrUpdate(EntityType.DEVICE, entity); + return entityId; + } + + protected UUID createView(String name) { + return createView(null, "default", name); + } + + protected UUID createView(CustomerId customerId, String name) { + return createView(customerId.getId(), "default", name); + } + + protected UUID createView(UUID customerId, String type, String name) { + UUID entityId = UUID.randomUUID(); + EntityView entity = new EntityView(); + entity.setId(new EntityViewId(entityId)); + entity.setType(type); + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ENTITY_VIEW, entity); + return entityId; + } + + protected UUID createEdge(String name) { + return createEdge(null, "default", name); + } + + protected UUID createEdge(CustomerId customerId, String name) { + return createEdge(customerId.getId(), "default", name); + } + + protected UUID createEdge(UUID customerId, String type, String name) { + UUID id = UUID.randomUUID(); + Edge edge = new Edge(); + edge.setId(new EdgeId(id)); + edge.setTenantId(tenantId); + if (customerId != null) { + edge.setCustomerId(new CustomerId(customerId)); + } + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + addOrUpdate(EntityType.EDGE, edge); + return id; + } + + protected UUID createSchedulerEvent(String type, EntityId originatorId, String name) { + return createSchedulerEvent(null, type, originatorId, name); + } + + protected UUID createSchedulerEvent(UUID customerId, String type, EntityId originatorId, String name) { + UUID id = UUID.randomUUID(); + SchedulerEvent schedulerEvent = new SchedulerEvent(); + schedulerEvent.setId(new SchedulerEventId(id)); + schedulerEvent.setTenantId(tenantId); + if (customerId != null) { + schedulerEvent.setCustomerId(new CustomerId(customerId)); + } + schedulerEvent.setType(type); + schedulerEvent.setName(name); + schedulerEvent.setConfiguration(JacksonUtil.newObjectNode()); + schedulerEvent.setSchedule(JacksonUtil.newObjectNode()); + schedulerEvent.setOriginatorId(originatorId); + schedulerEvent.setCreatedTime(42L); + addOrUpdate(EntityType.SCHEDULER_EVENT, schedulerEvent); + return id; + } + + protected UUID createAsset(String name) { + return createAsset(null, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, String name) { + return createAsset(customerId, defaultAssetProfileId, name); + } + + protected UUID createAsset(UUID customerId, UUID profileId, String name) { + UUID entityId = UUID.randomUUID(); + Asset entity = new Asset(); + entity.setId(new AssetId(entityId)); + if (profileId != null) { + entity.setAssetProfileId(new AssetProfileId(profileId)); + } + if (customerId != null) { + entity.setCustomerId(new CustomerId(customerId)); + } + entity.setName(name); + addOrUpdate(EntityType.ASSET, entity); + return entityId; + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, String type) { + createRelation(fromType, fromId, toType, toId, RelationTypeGroup.COMMON, type); + } + + protected void createRelation(EntityType fromType, UUID fromId, EntityType toType, UUID toId, RelationTypeGroup group, String type) { + repository.get(tenantId).addOrUpdate(new EntityRelation(EntityIdFactory.getByTypeAndUuid(fromType, fromId), EntityIdFactory.getByTypeAndUuid(toType, toId), type, group)); + } + + + protected boolean checkContains(PageData data, UUID entityId) { + return data.getData().stream().anyMatch(r -> r.getEntityId().getId().equals(entityId)); + } + + protected List createStringKeyFilters(String key, EntityKeyType keyType, StringFilterPredicate.StringOperation operation, String value) { + KeyFilter filter = new KeyFilter(); + filter.setKey(new EntityKey(keyType, key)); + filter.setValueType(EntityKeyValueType.STRING); + StringFilterPredicate predicate = new StringFilterPredicate(); + predicate.setValue(FilterPredicateValue.fromString(value)); + predicate.setOperation(operation); + predicate.setIgnoreCase(true); + filter.setPredicate(predicate); + return Collections.singletonList(filter); + } + + protected void addOrUpdate(EntityType entityType, Object entity) { + addOrUpdate(EdqsConverter.toEntity(entityType, entity)); + } + + protected void addOrUpdate(EdqsObject edqsObject) { + repository.get(tenantId).addOrUpdate(edqsObject); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java new file mode 100644 index 0000000000..de409ca293 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/ApiUsageStateFilterTest.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class ApiUsageStateFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + Tenant entity = new Tenant(); + entity.setId(tenantId); + entity.setTitle("test tenant"); + addOrUpdate(EntityType.TENANT, entity); + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerApiUsageState() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + + ApiUsageState apiUsageState = buildApiUsageState(customerId); + addOrUpdate(EntityType.API_USAGE_STATE, apiUsageState); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new CustomerId(customerId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private ApiUsageState buildApiUsageState(UUID customerId) { + ApiUsageState apiUsageState = new ApiUsageState(); + apiUsageState.setId(new ApiUsageStateId(UUID.randomUUID())); + apiUsageState.setTenantId(tenantId); + apiUsageState.setEntityId(new CustomerId(customerId)); + apiUsageState.setTransportState(ApiUsageStateValue.ENABLED); + apiUsageState.setReExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setJsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setTbelExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setDbStorageState(ApiUsageStateValue.ENABLED); + apiUsageState.setSmsExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setEmailExecState(ApiUsageStateValue.ENABLED); + apiUsageState.setAlarmExecState(ApiUsageStateValue.ENABLED); + return apiUsageState; + } + + private static EntityDataQuery getEntityDataQuery(CustomerId customerId) { + ApiUsageStateFilter filter = new ApiUsageStateFilter(); + filter.setCustomerId(customerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("Customer A")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java new file mode 100644 index 0000000000..2983ce57c3 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetSearchQueryFilterTest.java @@ -0,0 +1,223 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class AssetSearchQueryFilterTest extends AbstractEDQTest { + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(null, assetProfileId.getId(), "root"); + UUID asset1 = createAsset(null, assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(null, assetProfileId.getId(), "A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + + // find all assets with asset type = default + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all assets last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets last level only, level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("Office")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindTenantAssetsWithGroupPermissionsOnly() { + UUID eg1 = createGroup(EntityType.ASSET, "Group A"); + + UUID root = createAsset("root"); + UUID asset1 = createAsset("A1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID asset2 = createAsset("A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssets() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + UUID root = createAsset(customerId.getId(), assetProfileId.getId(), "root"); + UUID asset1 = createAsset(customerId.getId(), assetProfileId.getId(), "A1"); + UUID asset2 = createAsset(customerId.getId(), assetProfileId.getId(), "A2"); + UUID asset3 = createAsset(customerId.getId(), defaultAssetProfileId, "A3"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset3, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets of root asset with profile "Office" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("Office", "default")); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + Assert.assertTrue(checkContains(relationsResult, asset3)); + + // find all assets with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(root), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("Office")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerAssetsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.ASSET, "Group A"); + + UUID root = createAsset(customerId.getId(), defaultAssetProfileId, "root"); + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId,"A1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId,"A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + } + + @Test + public void testFindCustomerAssetsWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + UUID asset1 = createAsset(subCustomer.getId(), defaultAssetProfileId,"A1"); + UUID eg1 = createGroup(subCustomer.getId(), EntityType.ASSET, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ASSET, asset1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + UUID root = createAsset(customerId.getId(), defaultAssetProfileId, "root"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId,"A2"); + + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, asset2, "Contains"); + + // find all assets group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ASSET, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(root), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, asset1)); + Assert.assertTrue(checkContains(relationsResult, asset2)); + } + + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List assetTypes) { + AssetSearchQueryFilter filter = new AssetSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setAssetTypes(assetTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "A"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java new file mode 100644 index 0000000000..6048669d66 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AssetTypeFilterTest.java @@ -0,0 +1,187 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.AssetTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AssetTypeFilterTest extends AbstractEDQTest { + + private final AssetProfileId assetProfileId = new AssetProfileId(UUID.randomUUID()); + private final AssetProfileId assetProfileId2 = new AssetProfileId(UUID.randomUUID()); + private Asset asset; + private Asset asset2; + private Asset asset3; + + @Before + public void setUp() { + AssetProfile assetProfile = new AssetProfile(assetProfileId); + assetProfile.setName("Office"); + assetProfile.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile); + + AssetProfile assetProfile2 = new AssetProfile(assetProfileId2); + assetProfile2.setName("Street"); + assetProfile2.setDefault(false); + addOrUpdate(EntityType.ASSET_PROFILE, assetProfile2); + + asset = buildAsset(assetProfileId, "Office 1"); + asset2 = buildAsset(assetProfileId, "Office 2"); + asset3 = buildAsset(assetProfileId2, "Abbey Road"); + + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(EntityType.ASSET, asset2); + addOrUpdate(EntityType.ASSET, asset3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantAsset() { + // find asset with type "Office" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(Collections.singletonList("Office"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Office 1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(asset.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Office" and "Street" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office", "Street"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var third = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Abbey Road")).findAny(); + assertThat(third).isPresent(); + assertThat(third.get().getEntityId()).isEqualTo(asset3.getId()); + assertThat(third.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(asset.getCreatedTime())); + + // find asset with type "Supermarket" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with name "%Office%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "%Office%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with name "Office 1" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "Office 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find asset with name "%Super%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), "%Super%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find asset with key filter: name contains "Office" + KeyFilter containsNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "office", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset with key filter: name starts with "office" and matches case + KeyFilter startsWithNameFilter = getAssetNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "office", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerAsset() { + addOrUpdate(EntityType.ASSET, asset); + addOrUpdate(new LatestTsKv(asset.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + asset.setCustomerId(customerId); + addOrUpdate(EntityType.ASSET, asset); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Office"), null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(asset.getId(), first.getEntityId()); + Assert.assertEquals("Office 1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getAssetTypeQuery(List.of("Supermarket"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Asset buildAsset(AssetProfileId assetProfileId, String assetName) { + Asset asset = new Asset(); + asset.setId(new AssetId(UUID.randomUUID())); + asset.setTenantId(tenantId); + asset.setAssetProfileId(assetProfileId); + asset.setName(assetName); + asset.setCreatedTime(42L); + return asset; + } + + private static EntityDataQuery getAssetTypeQuery(List assetTypes, String assetNameRegex, List keyFilters) { + AssetTypeFilter filter = new AssetTypeFilter(); + filter.setAssetTypes(assetTypes); + filter.setAssetNameFilter(assetNameRegex); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getAssetNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java new file mode 100644 index 0000000000..bea0f57246 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceSearchQueryFilterTest.java @@ -0,0 +1,241 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class DeviceSearchQueryFilterTest extends AbstractEDQTest { + private final DeviceProfileId deviceProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset( "A1"); + UUID asset2 = createAsset( "A2"); + UUID device1 = createDevice(null, deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(null, deviceProfileId.getId(), "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of asset A1 + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all devices with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices with asset type = default + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all devices last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("thermostat")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device2)); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all devices last level only, level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, true, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindTenantDevicesWithGroupPermissionsOnly() { + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + + UUID asset1 = createAsset( "A1"); + UUID asset2 = createAsset( "A2"); + UUID device1 = createDevice(null, defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(null, defaultDeviceProfileId, "D2"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerDevices() { + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices of type "thermostat" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + + // find all assets of root asset with profile "Office" and "default" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat", "default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + + // find all assets with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerAssetsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + + DeviceProfile deviceProfile = new DeviceProfile(deviceProfileId); + deviceProfile.setName("thermostat"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(customerId.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), deviceProfileId.getId(), "D1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + } + + @Test + public void testFindCustomerEdgesWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID eg1 = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID asset2 = createAsset(subCustomer.getId(), defaultAssetProfileId, "A2"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, device1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + UUID device2 = createDevice(subCustomer.getId(), defaultDeviceProfileId, "D2"); + + createRelation(EntityType.ASSET, asset1, EntityType.ASSET, asset2, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset2, EntityType.DEVICE, device2, "Contains"); + + // find all devices with generic and group permission + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, subCustomer, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, device1)); + Assert.assertTrue(checkContains(relationsResult, device2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List deviceTypes) { + DeviceSearchQueryFilter filter = new DeviceSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setDeviceTypes(deviceTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "D"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java new file mode 100644 index 0000000000..e4c3a0ffad --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/DeviceTypeFilterTest.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +public class DeviceTypeFilterTest extends AbstractEDQTest { + + private final DeviceProfileId loraProfileId = new DeviceProfileId(UUID.randomUUID()); + + @Before + public void setUp() { + DeviceProfile deviceProfile = new DeviceProfile(loraProfileId); + deviceProfile.setName("LoRa"); + deviceProfile.setDefault(false); + deviceProfile.setType(DeviceProfileType.DEFAULT); + addOrUpdate(EntityType.DEVICE_PROFILE, deviceProfile); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setDeviceProfileId(loraProfileId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("default"), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(loraProfileId); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceTypeQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceTypeQuery(String deviceType) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(Collections.singletonList(deviceType)); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java new file mode 100644 index 0000000000..15a0b8b4e4 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeSearchQueryFilterTest.java @@ -0,0 +1,167 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public class EdgeSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindDevicesManagesByTenant() { + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID device3 = createDevice("D3"); + + createRelation(EntityType.EDGE, edge1, EntityType.DEVICE, device1, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device2, "Manages"); + createRelation(EntityType.EDGE, edge2, EntityType.DEVICE, device3, "Manages"); + + // find devices managed by edge + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + + // find devices managed by edge with non-existing type + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 1, false, Arrays.asList("non-existing type")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new DeviceId(device1), + EntitySearchDirection.TO, "Manages", 2, true, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindTenantEdgesWithGroupPermissionOnly() { + UUID eg1 = createGroup(EntityType.EDGE, "Group A"); + + UUID edge1 = createEdge("E1"); + UUID edge2 = createEdge("E2"); + createRelation(EntityType.TENANT, tenantId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.TENANT, tenantId.getId(), EntityType.EDGE, edge2, "Manages"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.EDGE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, tenantId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + } + + @Test + public void testFindCustomerEdges() { + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + + // find all edges managed by customer + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + + // find all edges managed by customer with non-existing type + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("non existing")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerEdgesWithGroupPermission() { + UUID eg1 = createGroup(EntityType.EDGE, "Group A"); + + UUID edge1 = createEdge(customerId, "E1"); + UUID edge2 = createEdge(customerId, "E2"); + UUID edge3 = createEdge(customerId, "E3"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, edge2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge1, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge2, "Manages"); + createRelation(EntityType.CUSTOMER, customerId.getId(), EntityType.EDGE, edge3, "Manages"); + + // find all entity views with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.EDGE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, customerId, + EntitySearchDirection.FROM, "Manages", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, edge1)); + Assert.assertTrue(checkContains(relationsResult, edge2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List edgeTypes) { + EdgeSearchQueryFilter filter = new EdgeSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEdgeTypes(edgeTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "E"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java new file mode 100644 index 0000000000..8c48e0f1ea --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EdgeTypeFilterTest.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EdgeTypeFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EdgeTypeFilterTest extends AbstractEDQTest { + + private Edge edge; + private Edge edge2; + private Edge edge3; + + + @Before + public void setUp() { + edge = buildEdge("default", "Edge 1"); + edge2 = buildEdge("default", "Edge 2"); + edge3 = buildEdge("edge v2", "Edge 3"); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEdges() { + // find edges with type "default" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(Collections.singletonList("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find edges with types "default" and "edge v2" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(Arrays.asList("default", "edge v2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 3")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(edge3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Edge%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "%Edge%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Edge 1" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "Edge 1", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Edge 4%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), "%Edge 4%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Edge" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Edge", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, List.of(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "edge" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "edge", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, List.of(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEdges() { + addOrUpdate(new LatestTsKv(edge.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + edge.setCustomerId(customerId); + edge2.setCustomerId(customerId); + edge3.setCustomerId(customerId); + addOrUpdate(EntityType.EDGE, edge); + addOrUpdate(EntityType.EDGE, edge2); + addOrUpdate(EntityType.EDGE, edge3); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("default"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("Edge 1")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(edge.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(edge.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEdgeTypeQuery(List.of("edge v3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Edge buildEdge(String type, String name) { + Edge edge = new Edge(); + edge.setId(new EdgeId(UUID.randomUUID())); + edge.setTenantId(tenantId); + edge.setType(type); + edge.setName(name); + edge.setCreatedTime(42L); + return edge; + } + + private static EntityDataQuery getEdgeTypeQuery(List edgeTypes, String edgeNameFilter, List keyFilters) { + EdgeTypeFilter filter = new EdgeTypeFilter(); + filter.setEdgeTypes(edgeTypes); + filter.setEdgeNameFilter(edgeNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java new file mode 100644 index 0000000000..0d0a623f7a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupIdFilterTest.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntitiesByGroupIdFilterTest extends AbstractEDQTest { + + private UUID deviceId; + private UUID deviceId2; + private UUID deviceId3; + + private UUID groupAId; + private UUID groupBId; + + @Before + public void setUp() { + deviceId = createDevice(customerId, "Lora-1"); + deviceId2 = createDevice(customerId, "Lora-2"); + deviceId3 = createDevice(customerId, "Lora-3"); + + // add device and device 2 to Group A + groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 3 to Group B + groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupBId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntitiesOfGroupA() { + // get devices of group A + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, deviceId)); + Assert.assertTrue(checkContains(result, deviceId2)); + + //get devices of non-existing group + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(UUID.randomUUID()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), List.of(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesOfGroupA() { + var result = repository.findEntityDataByQuery(tenantId, new CustomerId(UUID.randomUUID()), RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindCustomerEntitiesWithGroupPermission() { + MergedUserPermissions groupAPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupAPermission, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + + MergedUserPermissions groupBPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + result = repository.findEntityDataByQuery(tenantId, customerId, groupBPermission, getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupAId), null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Subcustomer A"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntitiesByGroupDataQuery(EntityType.DEVICE, new EntityGroupId(groupBId), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, deviceId3)); + } + + private static EntityDataQuery getEntitiesByGroupDataQuery(EntityType entityType, EntityGroupId groupId, List keyFilters) { + EntityGroupFilter filter = new EntityGroupFilter(); + filter.setGroupType(entityType); + filter.setEntityGroup(groupId.getId().toString()); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java new file mode 100644 index 0000000000..a6a81abd46 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntitiesByGroupNameFilterTest.java @@ -0,0 +1,140 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntitiesByGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntitiesByGroupNameFilterTest extends AbstractEDQTest { + + private UUID deviceId; + private UUID deviceId2; + private UUID deviceId3; + + private UUID groupAId; + private UUID groupBId; + + @Before + public void setUp() { + deviceId = createDevice(customerId, "Lora-1"); + deviceId2 = createDevice(customerId, "Lora-2"); + deviceId3 = createDevice(customerId, "Lora-3"); + + // add device and device 2 to Group A + groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 3 to Group B + groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupBId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntitiesOfGroupA() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group B", null, null), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntitiesOfGroupA() { + var result = repository.findEntityDataByQuery(tenantId, new CustomerId(UUID.randomUUID()), RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindCustomerEntitiesOfGroupAWithGroupPermission() { + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group A", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + List entityIds = result.getData().stream().map(queryResult -> queryResult.getEntityId().getId()).toList(); + assertThat(entityIds).containsOnly(deviceId, deviceId2); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Subcustomer A"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group B"); + UUID deviceId4 = createDevice(subCustomer, "Lora-4"); + createRelation(EntityType.ENTITY_GROUP, subCustomerGroupId, EntityType.DEVICE, deviceId4, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupBId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntitiesByGroupNameDataQuery(EntityType.DEVICE, "Group B", null, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + } + + private static EntityDataQuery getEntitiesByGroupNameDataQuery(EntityType entityType, String groupName, EntityId ownerId, List keyFilters) { + EntitiesByGroupNameFilter filter = new EntitiesByGroupNameFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupNameFilter(groupName); + filter.setOwnerId(ownerId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java new file mode 100644 index 0000000000..971b9d52ad --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupListFilterTest.java @@ -0,0 +1,189 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupListFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityGroupListFilterTest extends AbstractEDQTest { + + private EntityGroup deviceGroup; + private EntityGroup deviceGroup2; + private EntityGroup dashboardGroup; + + @Before + public void setUp() { + deviceGroup = buildEntityGroup(EntityType.DEVICE, "thermostats"); + deviceGroup2 = buildEntityGroup(EntityType.DEVICE, "humidity-sensors"); + dashboardGroup = buildEntityGroup(EntityType.DASHBOARD, "device dashboards"); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup2); + addOrUpdate(EntityType.ENTITY_GROUP, dashboardGroup); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityGroups() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals("thermostats", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE,List.of(deviceGroup.getId().getId().toString(), deviceGroup2.getId().getId().toString()), null), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString()), Arrays.asList(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindTenantEntityGroupsWithGroupPermissionOnly() { + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.DEVICE_GROUP, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, null, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityGroups() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + Assert.assertEquals(0, result.getTotalElements()); + + deviceGroup.setOwnerId(customerId); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDeviceGroupWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + EntityGroup deviceGroup3 = buildEntityGroup(EntityType.DEVICE, "sensors A"); + deviceGroup3.setOwnerId(subCustomer); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup3); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(deviceGroup3.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(deviceGroup3.getId().getId().toString()), null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup3.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup3.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup3.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID customerGroupId = createGroup(customerId.getId(), EntityType.DEVICE, "customer group"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "subcustomer group"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(customerGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, getEntityGroupListDataQuery(EntityType.DEVICE, List.of(subCustomerGroupId.toString(), customerGroupId.toString()), null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, customerGroupId)); + Assert.assertTrue(checkContains(result, subCustomerGroupId)); + } + + private EntityGroup buildEntityGroup(EntityType entityType, String name) { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(UUID.randomUUID())); + entityGroup.setTenantId(tenantId); + entityGroup.setOwnerId(tenantId); + entityGroup.setName(name); + entityGroup.setType(entityType); + entityGroup.setCreatedTime(42L); + return entityGroup; + } + + private static EntityDataQuery getEntityGroupListDataQuery(EntityType entityType, List ids, List keyFilters) { + EntityGroupListFilter filter = new EntityGroupListFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java new file mode 100644 index 0000000000..7ad296e946 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityGroupNameFilterTest.java @@ -0,0 +1,186 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityGroupNameFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityGroupNameFilterTest extends AbstractEDQTest { + + private EntityGroup deviceGroup; + private EntityGroup deviceGroup2; + private EntityGroup dashboardGroup; + + @Before + public void setUp() { + deviceGroup = buildEntityGroup(EntityType.DEVICE, "thermostats"); + deviceGroup2 = buildEntityGroup(EntityType.DEVICE, "thermostats 2"); + dashboardGroup = buildEntityGroup(EntityType.DASHBOARD, "device dashboards"); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup2); + addOrUpdate(EntityType.ENTITY_GROUP, dashboardGroup); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityGroups() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional group = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals(deviceGroup.getName())).findFirst(); + assertThat(group).isPresent(); + var first = group.get(); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermostats 2", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "humidity", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + //add name filter + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "humidity"); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", List.of(nameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityGroups() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + deviceGroup.setOwnerId(customerId); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityGroupNameDataQuery(EntityType.DEVICE, "thermo", null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindCustomerDeviceGroupWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + EntityGroup deviceGroup3 = buildEntityGroup(EntityType.DEVICE, "sensors A"); + deviceGroup3.setOwnerId(subCustomer); + addOrUpdate(EntityType.ENTITY_GROUP, deviceGroup3); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(deviceGroup3.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityGroupNameDataQuery(EntityType.DEVICE,"sensors", null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceGroup3.getId(), first.getEntityId()); + Assert.assertEquals(deviceGroup3.getName(), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals(String.valueOf(deviceGroup3.getCreatedTime()), first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindGroupWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + + UUID customerGroupId = createGroup(customerId.getId(), EntityType.DEVICE, "sensors A"); + UUID subCustomerGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "sensors A"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(customerGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, subCustomer, groupPermission, + getEntityGroupNameDataQuery(EntityType.DEVICE, "sensors A", null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, customerGroupId)); + Assert.assertTrue(checkContains(result, subCustomerGroupId)); + } + + private EntityGroup buildEntityGroup(EntityType entityType, String name) { + EntityGroup entityGroup = new EntityGroup(); + entityGroup.setId(new EntityGroupId(UUID.randomUUID())); + entityGroup.setTenantId(tenantId); + entityGroup.setOwnerId(tenantId); + entityGroup.setName(name); + entityGroup.setType(entityType); + entityGroup.setCreatedTime(42L); + return entityGroup; + } + + private static EntityDataQuery getEntityGroupNameDataQuery(EntityType entityType, String groupName, List keyFilters) { + EntityGroupNameFilter filter = new EntityGroupNameFilter(); + filter.setGroupType(entityType); + filter.setEntityGroupNameFilter(groupName); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + return new EntityDataQuery(filter, pageLink, entityFields, null, keyFilters); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java new file mode 100644 index 0000000000..d47252933f --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityListFilterTest.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.group.EntityGroup; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityListFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + private EntityGroup deviceGroup; + + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + // get entity list + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE,List.of(device.getId().getId().toString(), device2.getId().getId().toString())), false); + Assert.assertEquals(2, result.getTotalElements()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(UUID.randomUUID().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS,getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + @Test + public void testFindCustomerDeviceWithGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + device.setCustomerId(subCustomer); + addOrUpdate(EntityType.DEVICE, device); + + // add device to customer device group + UUID deviceGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, deviceGroupId, EntityType.DEVICE, device.getUuidId(), RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Collections.emptyMap(), Map.of(new EntityGroupId(deviceGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + @Test + public void testFindCustomerDeviceWithGenericAndGroupPermission() { + CustomerId subCustomer = new CustomerId(UUID.randomUUID()); + createCustomer(subCustomer.getId(), customerId.getId(), "Sub Customer A"); + device.setCustomerId(subCustomer); + addOrUpdate(EntityType.DEVICE, device); + + // add device to customer device group + UUID deviceGroupId = createGroup(subCustomer.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, deviceGroupId, EntityType.DEVICE, device.getUuidId(), RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions groupPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(deviceGroupId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, customerId, groupPermission, getEntityListDataQuery(EntityType.DEVICE, List.of(device.getId().getId().toString())), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityListDataQuery(EntityType entityType, List ids) { + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entityType); + filter.setEntityList(ids); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = getNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "LoRa-"); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + + private static KeyFilter getNameKeyFilter(StringFilterPredicate.StringOperation operation, String value) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(value)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java new file mode 100644 index 0000000000..8b17c30f2d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityNameFilterTest.java @@ -0,0 +1,132 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityNameFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class EntityNameFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("Not LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("%1"), false); + Assert.assertEquals(1, result.getTotalElements()); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("L%"), false); + Assert.assertEquals(1, result.getTotalElements()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getDeviceNameQuery("LoRa"), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getDeviceNameQuery(String entityNameFilter) { + EntityNameFilter filter = new EntityNameFilter(); + filter.setEntityType(EntityType.DEVICE); + filter.setEntityNameFilter(entityNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java new file mode 100644 index 0000000000..a2b53b1f7d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityTypeFilterTest.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public class EntityTypeFilterTest extends AbstractEDQTest { + + private Device device; + private Device device2; + private Device device3; + + @Before + public void setUp() { + device = buildDevice("LoRa-1"); + device2 = buildDevice("LoRa-2"); + device3 = buildDevice("Parking-Sensor-1"); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(EntityType.DEVICE, device2); + addOrUpdate(EntityType.DEVICE, device3); + addOrUpdate(new LatestTsKv(device.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "enabled")), 0L)); + addOrUpdate(new LatestTsKv(device2.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "disabled")), 0L)); + addOrUpdate(new LatestTsKv(device3.getId(), new BasicTsKvEntry(43, new BooleanDataEntry("free", true)), 0L)); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDeviceEntities() { + // find all tenant devices + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + var first = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("LoRa-1")).findAny(); + assertThat(first).isPresent(); + assertThat(first.get().getEntityId()).isEqualTo(device.getId()); + assertThat(first.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(device.getCreatedTime())); + assertThat(first.get().getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()).isEqualTo("enabled"); + + // find all tenant devices with filter by name + KeyFilter keyFilter = getDeviceNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, List.of(keyFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find asset entities + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerDeviceEntities() { + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.DEVICE, null), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(device.getId(), first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + Assert.assertEquals("enabled", first.getLatest().get(EntityKeyType.TIME_SERIES).get("state").getValue()); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityTypeQuery(EntityType.ASSET, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private Device buildDevice(String name) { + Device device = new Device(); + device.setId(new DeviceId(UUID.randomUUID())); + device.setTenantId(tenantId); + device.setName(name); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + return device; + } + + private static EntityDataQuery getEntityTypeQuery(EntityType entityType, List keyFilters) { + EntityTypeFilter filter = new EntityTypeFilter(); + filter.setEntityType(entityType); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getDeviceNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java new file mode 100644 index 0000000000..b984fe76b9 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewSearchQueryFilterTest.java @@ -0,0 +1,221 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityViewSearchQueryFilter; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class EntityViewSearchQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantEntityViews() { + UUID asset1 = createAsset( "A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of asset A1 + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views with max level = 1 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("default")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with type "day 1" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 1, false, Arrays.asList("day 1")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views last level only, level = 2 + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, true, Arrays.asList("default")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + @Test + public void testFindTenantDevicesWithGroupPermissionOnly() { + UUID eg1 = createGroup(EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset( "A1"); + UUID device1 = createDevice("D1"); + UUID device2 = createDevice("D2"); + UUID deviceView1 = createView("V1"); + UUID deviceView2 = createView("V2"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all devices with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, null, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("default")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + } + + @Test + public void testFindCustomerDevices() { + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views of type "day 1" + PageData relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + + // find all entity views of type "day 2" + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 2")); + Assert.assertEquals(0, relationsResult.getData().size()); + + // find all entity views with other customer + relationsResult = findData(RepositoryUtils.ALL_READ_PERMISSIONS, new CustomerId(UUID.randomUUID()), new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("thermostat")); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerEntityViewsWithGroupPermission() { + UUID eg1 = createGroup(EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerId.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerId.getId(), "day 1", "V2"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views with group permission only + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + } + + @Test + public void testFindCustomerAssetsWithGenericAndGroupPermission() { + CustomerId customerB = new CustomerId(UUID.randomUUID()); + createCustomer(customerB.getId(), customerId.getId(), "Customer B"); + + UUID eg1 = createGroup(customerB.getId(), EntityType.ENTITY_VIEW, "Group A"); + + UUID asset1 = createAsset(customerId.getId(), defaultAssetProfileId, "A1"); + UUID device1 = createDevice(customerId.getId(), defaultDeviceProfileId, "D1"); + UUID device2 = createDevice(customerB.getId(), defaultDeviceProfileId,"D2"); + UUID deviceView1 = createView(customerId.getId(),"day 1", "V1"); + UUID deviceView2 = createView(customerB.getId(), "day 1", "V2"); + + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.ENTITY_VIEW, deviceView1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device1, "Contains"); + createRelation(EntityType.ASSET, asset1, EntityType.DEVICE, device2, "Contains"); + createRelation(EntityType.DEVICE, device1, EntityType.ENTITY_VIEW, deviceView1, "Contains"); + createRelation(EntityType.DEVICE, device2, EntityType.ENTITY_VIEW, deviceView2, "Contains"); + + // find all entity views with generic and group permission + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.ENTITY_VIEW, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + PageData relationsResult = findData(readGroupPermissions, customerId, new AssetId(asset1), + EntitySearchDirection.FROM, "Contains", 2, false, Arrays.asList("day 1")); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, deviceView1)); + Assert.assertTrue(checkContains(relationsResult, deviceView2)); + } + + private PageData findData(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, + EntitySearchDirection direction, String relationType, int maxLevel, boolean lastLevelOnly, List entityViewTypes) { + EntityViewSearchQueryFilter filter = new EntityViewSearchQueryFilter(); + filter.setRootEntity(rootId); + filter.setDirection(direction); + filter.setRelationType(relationType); + filter.setEntityViewTypes(entityViewTypes); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "V"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java new file mode 100644 index 0000000000..4b086f1ef5 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/EntityViewTypeFilterTest.java @@ -0,0 +1,177 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.EntityViewTypeFilter; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class EntityViewTypeFilterTest extends AbstractEDQTest { + + private EntityView entityView; + private EntityView entityView2; + private EntityView entityView3; + + + @Before + public void setUp() { + entityView = buildEntityView("day 1", "day 1 lora 1 view"); + entityView2 = buildEntityView("day 1", "day 1 lora 2 view"); + entityView3 = buildEntityView("day 2", "day 2 lora 1 view"); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantEntityView() { + // find entity view with type "day 1" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with types "day 1" and "day 2" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Arrays.asList("day 1", "day 2"), null, null), false); + + Assert.assertEquals(3, result.getTotalElements()); + Optional thirdView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 2 lora 1 view")).findFirst(); + assertThat(thirdView).isPresent(); + assertThat(thirdView.get().getEntityId()).isEqualTo(entityView3.getId()); + assertThat(thirdView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + // find entity view with type "day 3" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with name "%Lora%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 1 lora%", null), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with name "Lora 1 device view" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "day 1 lora 1 view", null), false); + Assert.assertEquals(1, result.getTotalElements()); + + // find entity view with name "%Parking sensor%" + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), "%day 3 lora%", null), false); + Assert.assertEquals(0, result.getTotalElements()); + + // find entity view with key filter: name contains "Lora" + KeyFilter containsNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "Lora", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(containsNameFilter)), false); + Assert.assertEquals(2, result.getTotalElements()); + + // find entity view with key filter: name starts with "lora" and matches case + KeyFilter startsWithNameFilter = getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation.STARTS_WITH, "lora", false); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, Arrays.asList(startsWithNameFilter)), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindCustomerEntityView() { + addOrUpdate(new LatestTsKv(entityView.getId(), new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + + entityView.setCustomerId(customerId); + entityView2.setCustomerId(customerId); + entityView3.setCustomerId(customerId); + addOrUpdate(EntityType.ENTITY_VIEW, entityView); + addOrUpdate(EntityType.ENTITY_VIEW, entityView2); + addOrUpdate(EntityType.ENTITY_VIEW, entityView3); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 1"), null, null), false); + + Assert.assertEquals(2, result.getTotalElements()); + Optional firstView = result.getData().stream().filter(queryResult -> queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue().equals("day 1 lora 1 view")).findFirst(); + assertThat(firstView).isPresent(); + assertThat(firstView.get().getEntityId()).isEqualTo(entityView.getId()); + assertThat(firstView.get().getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()).isEqualTo(String.valueOf(entityView.getCreatedTime())); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityViewTypeQuery(Collections.singletonList("day 3"), null, null), false); + Assert.assertEquals(0, result.getTotalElements()); + } + + private EntityView buildEntityView(String type, String name) { + EntityView entityView = new EntityView(); + entityView.setId(new EntityViewId(UUID.randomUUID())); + entityView.setTenantId(tenantId); + entityView.setType(type); + entityView.setName(name); + entityView.setCreatedTime(42L); + return entityView; + } + + private static EntityDataQuery getEntityViewTypeQuery(List assetTypes, String entityViewNameFilter, List keyFilters) { + EntityViewTypeFilter filter = new EntityViewTypeFilter(); + filter.setEntityViewTypes(assetTypes); + filter.setEntityViewNameFilter(entityViewNameFilter); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getEntityViewNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java new file mode 100644 index 0000000000..b0a90d9dce --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RelationsQueryFilterTest.java @@ -0,0 +1,233 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; + +public class RelationsQueryFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @Test + public void testFindTenantDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice("NOT MATCHING D3"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da3, "Contains"); + + PageData relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta2)); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + @Test + public void testFindTenantDevicesLastLevelOnly() { + UUID root = createAsset("T ROOT"); + + UUID ta1 = createAsset("T A1 NO MORE RELATIONS"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + UUID da4 = createDevice(customerId, "T D4"); // Lvl 4 + + // ROOT --Contains--> A1, A2; A2 --Contains--> D1, D2; D2 --Contains--> D3. + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta1, "Contains"); + createRelation(EntityType.ASSET, root, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta2, EntityType.DEVICE, da2, "Contains"); + createRelation(EntityType.ASSET, da2, EntityType.DEVICE, da3, "Contains"); + createRelation(EntityType.ASSET, da3, EntityType.DEVICE, da4, "Contains"); + + PageData relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 1, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, ta2)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 2, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da2)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 3, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(RepositoryUtils.ALL_READ_PERMISSIONS, null, new AssetId(root), 4, true, + new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(3, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, ta1)); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da4)); + + } + + @Test + public void testFindTenantDevicesGroupsOnly() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice("T D1"); + UUID da2 = createDevice(customerId, "T D2"); + + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + + PageData relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(readGroupPermissions, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerDevices() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice("T D2"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + + PageData relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + + relationsResult = filter(customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(0, relationsResult.getData().size()); + } + + @Test + public void testFindCustomerDevicesGroupsOnly() { + UUID ta1 = createAsset("T A1"); + UUID ta2 = createAsset("T A2"); + UUID da1 = createDevice(customerId, "T D1"); + UUID da2 = createDevice(customerId, "T D2"); + UUID da3 = createDevice(customerId, "T D3"); + + UUID eg1 = createGroup(EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da1, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, eg1, EntityType.DEVICE, da3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // A1 --Contains--> A2, A1 --Contains--> D1. A1 --Manages--> D2. + createRelation(EntityType.ASSET, ta1, EntityType.ASSET, ta2, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da1, "Contains"); + createRelation(EntityType.ASSET, ta1, EntityType.DEVICE, da2, "Manages"); + createRelation(EntityType.DEVICE, da2, EntityType.DEVICE, da3, "Contains"); + + MergedUserPermissions readGroupPermissions = new MergedUserPermissions(Collections.emptyMap(), Collections.singletonMap(new EntityGroupId(eg1), + new MergedGroupPermissionInfo(EntityType.DEVICE, new HashSet<>(Arrays.asList(Operation.READ, Operation.READ_ATTRIBUTES, Operation.READ_TELEMETRY))))); + + PageData relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Contains", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(2, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da1)); + Assert.assertTrue(checkContains(relationsResult, da3)); + + relationsResult = filter(readGroupPermissions, customerId, new AssetId(ta1), new RelationEntityTypeFilter("Manages", Arrays.asList(EntityType.DEVICE, EntityType.ASSET))); + Assert.assertEquals(1, relationsResult.getData().size()); + Assert.assertTrue(checkContains(relationsResult, da2)); + } + + private PageData filter(EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(RepositoryUtils.ALL_READ_PERMISSIONS, rootId, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(permissions, null, rootId, relationEntityTypeFilters); + } + + private PageData filter(CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(RepositoryUtils.ALL_READ_PERMISSIONS, customerId, rootId, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, RelationEntityTypeFilter... relationEntityTypeFilters) { + return filter(permissions, customerId, rootId, 3, false, relationEntityTypeFilters); + } + + private PageData filter(MergedUserPermissions permissions, CustomerId customerId, EntityId rootId, int maxLevel, boolean lastLevelOnly, RelationEntityTypeFilter... relationEntityTypeFilters) { + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setRootEntity(rootId); + filter.setFilters(Arrays.asList(relationEntityTypeFilters)); + filter.setDirection(EntitySearchDirection.FROM); + filter.setFetchLastLevelOnly(lastLevelOnly); + filter.setMaxLevel(maxLevel); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringFilterPredicate.StringOperation.STARTS_WITH, "T"); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + return repository.findEntityDataByQuery(tenantId, customerId, permissions, query, false); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java new file mode 100644 index 0000000000..42706a796a --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/RepositoryUtilsTest.java @@ -0,0 +1,434 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.edqs.fields.DeviceFields; +import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate.BooleanOperation; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate; +import org.thingsboard.server.common.data.query.ComplexFilterPredicate.ComplexOperation; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.NumericFilterPredicate; +import org.thingsboard.server.common.data.query.NumericFilterPredicate.NumericOperation; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; +import org.thingsboard.server.edqs.data.DeviceData; +import org.thingsboard.server.edqs.data.EntityProfileData; +import org.thingsboard.server.edqs.data.dp.BoolDataPoint; +import org.thingsboard.server.edqs.data.dp.DoubleDataPoint; +import org.thingsboard.server.edqs.data.dp.LongDataPoint; +import org.thingsboard.server.edqs.data.dp.StringDataPoint; +import org.thingsboard.server.edqs.query.DataKey; +import org.thingsboard.server.edqs.query.EdqsFilter; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class RepositoryUtilsTest { + + private static Stream deviceNameFilters() { + return Stream.of(Arguments.of(null, getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet device 123", getNameFilter(StringOperation.STARTS_WITH, "lora"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.STARTS_WITH, "ra"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.ENDS_WITH, "device"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet 123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.EQUAL, "loranet "), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_EQUAL, "loranet 123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.CONTAINS, "loranet123"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet123"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_CONTAINS, "loranet"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 123, loranet 124"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.IN, "loranet 125, loranet 126"), false), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 125, loranet 126"), true), + Arguments.of("loranet 123", getNameFilter(StringOperation.NOT_IN, "loranet 123, loranet 126"), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameFilters") + public void testFilterByDeviceName(String deviceName, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream createdTimeFilters() { + return Stream.of(Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.NOT_EQUAL, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 999), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.GREATER_OR_EQUAL, 1001), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1001), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS, 1000), false), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 1000), true), + Arguments.of(1000, getCreatedTimeFilter(NumericOperation.LESS_OR_EQUAL, 999), false) + ); + } + + @ParameterizedTest + @MethodSource("createdTimeFilters") + public void testFilterDevicesByCreatedTime(long createdTime, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().createdTime(createdTime).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceNameAndTypeFilter() { + return Stream.of( + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "thermostat")), true), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "thermostat")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "lo"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false), + Arguments.of("loranet 123", "thermostat", List.of(getNameFilter(StringOperation.STARTS_WITH, "net"), getTypeFilter(StringOperation.EQUAL, "sensor1")), false)); + } + + @ParameterizedTest + @MethodSource("deviceNameAndTypeFilter") + public void testFilterByDeviceNameAndDeviceType(String deviceName, String deviceType, List keyFilters, boolean result) { + UUID deviceProfileId = UUID.randomUUID(); + EntityProfileData deviceProfile = new EntityProfileData(deviceProfileId, EntityType.DEVICE_PROFILE); + deviceProfile.setFields(DeviceProfileFields.builder().name(deviceType).build()); + + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).deviceProfileId(deviceProfileId).type(deviceType).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceNameComplexFilters() { + return Stream.of(Arguments.of(null, List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "net")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the")), false), + Arguments.of("loranet123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.OR, StringOperation.ENDS_WITH, "123")), true), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "net", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "123")), false), + Arguments.of("loranet 123", List.of(getComplexComplexDeviceNameFilter(StringOperation.STARTS_WITH, "lo", ComplexOperation.OR, StringOperation.STARTS_WITH, "the", + ComplexOperation.AND, StringOperation.ENDS_WITH, "124")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceNameComplexFilters") + public void testFilterByDeviceNameComplexFilters(String deviceName, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(deviceName).build()); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceTemperatureFilters() { + return Stream.of(Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.EQUAL, 22.9), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.8), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.NOT_EQUAL, 22.9), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 22.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 23.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 23.0), true), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.0), false), + Arguments.of(22.8, getTemperatureFilter(NumericOperation.LESS_OR_EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureFilters") + public void testFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureComplexFilters() { + return Stream.of(Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.8, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), true), + Arguments.of(22.8, getComplexTemperatureFilter(NumericOperation.GREATER, 23.5, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30), false), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.OR, NumericOperation.GREATER, 35), true), + Arguments.of(22.8, getComplexComplexTemperatureFilter(NumericOperation.GREATER, 22.0, ComplexOperation.AND, NumericOperation.LESS_OR_EQUAL, 30, ComplexOperation.AND, NumericOperation.EQUAL, 22.8), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureComplexFilters") + public void testComplexFilterByDeviceTemperature(double tempValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceHumidityFilters() { + return Stream.of(Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.NOT_EQUAL, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 59), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60), true), + Arguments.of(60, getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 61), true), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS, 60), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 59), false), + Arguments.of(60, getHumidityFilter(NumericOperation.LESS_OR_EQUAL, 60), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceHumidityFilters") + public void testFilterByDeviceHumidity(long humidityValue, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceTemperatureAndHumidityFilters() { + return Stream.of(Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.EQUAL, 60)), true), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.EQUAL, 22.8), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER, 23), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 60)), false), + Arguments.of(22.8, 60, List.of(getTemperatureFilter(NumericOperation.GREATER_OR_EQUAL, 22.9), getHumidityFilter(NumericOperation.GREATER_OR_EQUAL, 61)), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceTemperatureAndHumidityFilters") + public void testFilterByDeviceTemperatureAndHumidity(double tempValue, long humidityValue, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putTs(5, new DoubleDataPoint(System.currentTimeMillis(), tempValue)); + deviceData.putTs(6, new LongDataPoint(System.currentTimeMillis(), humidityValue)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static Stream deviceVersionAttributeFilters() { + return Stream.of(Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, true), true), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.EQUAL, false), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, true), false), + Arguments.of(true, getActiveAttributeFilter(BooleanOperation.NOT_EQUAL, false), true) + ); + } + + @ParameterizedTest + @MethodSource("deviceVersionAttributeFilters") + public void testFilterByDeviceVersionAttribute(Boolean active, EdqsFilter keyFilter, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, List.of(keyFilter))).isEqualTo(result); + } + + private static Stream deviceActiveAndVersionFilters() { + return Stream.of(Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), true), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, true), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.1")), false), + Arguments.of(true, "3.2.1", List.of(getActiveAttributeFilter(BooleanOperation.EQUAL, false), getVersionAttributeFilter(StringOperation.EQUAL, "3.2.2")), false) + ); + } + + @ParameterizedTest + @MethodSource("deviceActiveAndVersionFilters") + public void testFilterByActiveAndVersionAttributes(Boolean active, String version, List keyFilters, boolean result) { + DeviceData deviceData = new DeviceData(UUID.randomUUID()); + deviceData.setCustomerId(UUID.randomUUID()); + deviceData.setFields(DeviceFields.builder().name(StringUtils.randomAlphabetic(10)).build()); + deviceData.putAttr(1, AttributeScope.CLIENT_SCOPE, new StringDataPoint(System.currentTimeMillis(), version)); + deviceData.putAttr(2, AttributeScope.SERVER_SCOPE, new BoolDataPoint(System.currentTimeMillis(), active)); + + assertThat(RepositoryUtils.checkKeyFilters(deviceData, keyFilters)).isEqualTo(result); + } + + private static EdqsFilter getVersionAttributeFilter(StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.CLIENT_ATTRIBUTE, "version", 1); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + + private static EdqsFilter getActiveAttributeFilter(BooleanOperation operation, boolean predicateValue) { + BooleanFilterPredicate filterPredicate = new BooleanFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromBoolean(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.SERVER_ATTRIBUTE, "active", 2); + return new EdqsFilter(key, EntityKeyValueType.BOOLEAN, filterPredicate); + } + + private static EdqsFilter getTemperatureFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("temperature", 5, operation, predicateValue); + } + + private static EdqsFilter getHumidityFilter(NumericOperation operation, double predicateValue) { + return getTimeseriesFilter("humidity", 6, operation, predicateValue); + } + + private static EdqsFilter getTimeseriesFilter(String key, Integer keysId, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey newKey = new DataKey(EntityKeyType.TIME_SERIES, key, keysId); + return new EdqsFilter(newKey, EntityKeyValueType.NUMERIC, filterPredicate); + } + + private static EdqsFilter getNameFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("name", operation, predicateValue); + } + + private static EdqsFilter getTypeFilter(StringOperation operation, String predicateValue) { + return getStringEntityFieldFilter("type", operation, predicateValue); + } + + private static EdqsFilter getStringEntityFieldFilter(String fieldName, StringOperation operation, String predicateValue) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, filterPredicate); + } + + private static EdqsFilter getCreatedTimeFilter(NumericOperation operation, double predicateValue) { + return getDatetimeEntityFieldFilter("createdTime", operation, predicateValue); + } + + private static EdqsFilter getDatetimeEntityFieldFilter(String fieldName, NumericOperation operation, double predicateValue) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, fieldName, 3); + return new EdqsFilter(key, EntityKeyValueType.DATE_TIME, filterPredicate); + } + + private static EdqsFilter getComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexTemperatureFilter(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2, + ComplexOperation complexOperation2, NumericOperation operation3, double predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexNumericFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.TIME_SERIES, "temperature", 5); + return new EdqsFilter(key, EntityKeyValueType.NUMERIC, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexNumericFilterPredicate(NumericOperation operation, double predicateValue, ComplexOperation complexOperation, NumericOperation operation2, double predicateValue2) { + NumericFilterPredicate filterPredicate = new NumericFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromDouble(predicateValue)); + + NumericFilterPredicate filterPredicate2 = new NumericFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromDouble(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, complexFilterPredicate); + } + + private static EdqsFilter getComplexComplexDeviceNameFilter(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2, + ComplexOperation complexOperation2, StringOperation operation3, String predicateValue3) { + ComplexFilterPredicate complexFilterPredicate = getComplexStringFilterPredicate(operation, predicateValue, complexOperation, operation2, predicateValue2); + + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation3); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue3)); + + ComplexFilterPredicate mainComplexFilterPredicate = new ComplexFilterPredicate(); + mainComplexFilterPredicate.setOperation(complexOperation2); + mainComplexFilterPredicate.setPredicates(List.of(complexFilterPredicate, filterPredicate)); + + DataKey key = new DataKey(EntityKeyType.ENTITY_FIELD, "name", 3); + return new EdqsFilter(key, EntityKeyValueType.STRING, mainComplexFilterPredicate); + } + + private static ComplexFilterPredicate getComplexStringFilterPredicate(StringOperation operation, String predicateValue, ComplexOperation complexOperation, StringOperation operation2, String predicateValue2) { + StringFilterPredicate filterPredicate = new StringFilterPredicate(); + filterPredicate.setOperation(operation); + filterPredicate.setValue(FilterPredicateValue.fromString(predicateValue)); + + StringFilterPredicate filterPredicate2 = new StringFilterPredicate(); + filterPredicate2.setOperation(operation2); + filterPredicate2.setValue(FilterPredicateValue.fromString(predicateValue2)); + + ComplexFilterPredicate complexFilterPredicate = new ComplexFilterPredicate(); + complexFilterPredicate.setOperation(complexOperation); + complexFilterPredicate.setPredicates(List.of(filterPredicate, filterPredicate2)); + return complexFilterPredicate; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java new file mode 100644 index 0000000000..8ebbff534d --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SchedulerEventFilterTest.java @@ -0,0 +1,129 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SchedulerEventFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class SchedulerEventFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantSchedulerEvents() { + UUID dashboardId = createDashboard("test dashboard"); + UUID deviceId = createDevice("test device"); + + UUID eventId1 = createSchedulerEvent("Update attributes", new DeviceId(deviceId), "Turn off device"); + UUID eventId2 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate morning report"); + UUID eventId3 = createSchedulerEvent("Generate report", new DashboardId(dashboardId), "Generate evening report"); + + // find all scheduler events with type "Generate report" + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + Assert.assertTrue(checkContains(result, eventId3)); + + // find all scheduler events for device originator + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId1)); + + // find all scheduler events with name "%morning%" + KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + } + + @Test + public void testFindCustomerEdges() { + UUID dashboardId = createDashboard( "test dashboard"); + UUID deviceId = createDevice("test device"); + + UUID eventId1 = createSchedulerEvent(customerId.getId(), "Update attributes", new DeviceId(deviceId), "Turn off device"); + UUID eventId2 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate morning report"); + UUID eventId3 = createSchedulerEvent(customerId.getId(), "Generate report", new DashboardId(dashboardId), "Generate evening report"); + + // find all scheduler events with type "Generate report" + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery("Generate report", null, null), false); + Assert.assertEquals(2, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + Assert.assertTrue(checkContains(result, eventId3)); + + // find all scheduler events for device originator + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, new DeviceId(deviceId), null), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId1)); + + // find all scheduler events with name "%morning%" + KeyFilter containsNameFilter = getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation.CONTAINS, "morning", true); + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getSchedulerEventQuery(null, null, List.of(containsNameFilter)), false); + Assert.assertEquals(1, result.getTotalElements()); + Assert.assertTrue(checkContains(result, eventId2)); + } + + private static EntityDataQuery getSchedulerEventQuery(String eventType, EntityId entityId, List keyFilters) { + SchedulerEventFilter filter = new SchedulerEventFilter(); + filter.setEventType(eventType); + filter.setOriginator(entityId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFilters); + } + + private static KeyFilter getSchedulerEventNameKeyFilter(StringFilterPredicate.StringOperation operation, String predicateValue, boolean ignoreCase) { + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(ignoreCase); + predicate.setOperation(operation); + predicate.setValue(new FilterPredicateValue<>(predicateValue)); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + return nameFilter; + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java new file mode 100644 index 0000000000..8755191e39 --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/SingleEntityFilterTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edqs.LatestTsKv; +import org.thingsboard.server.common.data.edqs.query.QueryResult; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityGroupId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.permission.MergedGroupPermissionInfo; +import org.thingsboard.server.common.data.permission.MergedUserPermissions; +import org.thingsboard.server.common.data.permission.Operation; +import org.thingsboard.server.common.data.permission.Resource; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public class SingleEntityFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindTenantDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(UUID.randomUUID())), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(1, result.getTotalElements()); + first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + @Test + public void testFindTenantDeviceWithGenericAndGroupPermission() { + UUID deviceId = createDevice(customerId, "LoRa-customer-1"); + UUID deviceId2 = createDevice(customerId, "LoRa-customer-2"); + UUID deviceId3 = createDevice(customerId, "LoRa-customer-3"); + + // add device and device 2 to Group A + UUID groupAId = createGroup(customerId.getId(), EntityType.DEVICE, "Group A"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId2, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + // add device and device 2 to Group A + UUID groupBId = createGroup(customerId.getId(), EntityType.DEVICE, "Group B"); + createRelation(EntityType.ENTITY_GROUP, groupAId, EntityType.DEVICE, deviceId3, RelationTypeGroup.FROM_ENTITY_GROUP, "Contains"); + + MergedUserPermissions genericAndGroupAPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + var result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupAPermission, getEntityDataQuery(new DeviceId(deviceId2)), false); + Assert.assertEquals(1, result.getTotalElements()); + QueryResult queryResult = result.getData().get(0); + Assert.assertEquals(deviceId2, queryResult.getEntityId().getId()); + Assert.assertEquals("LoRa-customer-2", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + + // find device without permission + MergedUserPermissions genericAndGroupBPermission = new MergedUserPermissions( + Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(new EntityGroupId(groupAId), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + result = repository.findEntityDataByQuery(tenantId, null, genericAndGroupBPermission, getEntityDataQuery(new DeviceId(deviceId3)), false); + Assert.assertEquals(1, result.getTotalElements()); + queryResult = result.getData().get(0); + Assert.assertEquals(deviceId3, queryResult.getEntityId().getId()); + Assert.assertEquals("LoRa-customer-3", queryResult.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + @Test + public void testFindCustomerDevice() { + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + Device device = new Device(); + device.setId(deviceId); + device.setTenantId(tenantId); + device.setName("LoRa-1"); + device.setCreatedTime(42L); + device.setDeviceProfileId(new DeviceProfileId(defaultDeviceProfileId)); + addOrUpdate(EntityType.DEVICE, device); + addOrUpdate(new LatestTsKv(deviceId, new BasicTsKvEntry(43, new StringDataEntry("state", "TEST")), 0L)); + + var result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + Assert.assertEquals(0, result.getTotalElements()); + + device.setCustomerId(customerId); + addOrUpdate(EntityType.DEVICE, device); + + result = repository.findEntityDataByQuery(tenantId, customerId, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(device.getId()), false); + + Assert.assertEquals(1, result.getTotalElements()); + var first = result.getData().get(0); + Assert.assertEquals(deviceId, first.getEntityId()); + Assert.assertEquals("LoRa-1", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + Assert.assertEquals("42", first.getLatest().get(EntityKeyType.ENTITY_FIELD).get("createdTime").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + SingleEntityFilter filter = new SingleEntityFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "state"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + var latestValues = Arrays.asList(new EntityKey(EntityKeyType.TIME_SERIES, "state")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, latestValues, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java new file mode 100644 index 0000000000..f0bb4c4e0b --- /dev/null +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/StateEntityOwnerFilterTest.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2024 ThingsBoard, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.repo; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.data.query.KeyFilter; +import org.thingsboard.server.common.data.query.StateEntityOwnerFilter; +import org.thingsboard.server.common.data.query.StringFilterPredicate; +import org.thingsboard.server.edqs.util.RepositoryUtils; + +import java.util.Arrays; +import java.util.UUID; + +public class StateEntityOwnerFilterTest extends AbstractEDQTest { + + @Before + public void setUp() { + } + + @After + public void tearDown() { + } + + @Test + public void testFindCustomerDeviceOwner() { + UUID customerId = UUID.randomUUID(); + createCustomer(customerId, null, "Customer A"); + UUID deviceId = createDevice(new CustomerId(customerId), "LoRa-1"); + + var result = repository.findEntityDataByQuery(tenantId, null, RepositoryUtils.ALL_READ_PERMISSIONS, getEntityDataQuery(new DeviceId(deviceId)), false); + + Assert.assertEquals(1, result.getTotalElements()); + var customer = result.getData().get(0); + Assert.assertEquals(customerId, customer.getEntityId().getId()); + Assert.assertEquals("Customer A", customer.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()); + } + + private static EntityDataQuery getEntityDataQuery(DeviceId deviceId) { + StateEntityOwnerFilter filter = new StateEntityOwnerFilter(); + filter.setSingleEntity(deviceId); + var pageLink = new EntityDataPageLink(20, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.TIME_SERIES, "name"), EntityDataSortOrder.Direction.DESC), false); + + var entityFields = Arrays.asList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"), new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime")); + KeyFilter nameFilter = new KeyFilter(); + nameFilter.setKey(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + var predicate = new StringFilterPredicate(); + predicate.setIgnoreCase(false); + predicate.setOperation(StringFilterPredicate.StringOperation.CONTAINS); + predicate.setValue(new FilterPredicateValue<>("LoRa-")); + nameFilter.setPredicate(predicate); + nameFilter.setValueType(EntityKeyValueType.STRING); + + return new EntityDataQuery(filter, pageLink, entityFields, null, Arrays.asList(nameFilter)); + } + +} diff --git a/edqs/src/test/resources/edq-test.properties b/edqs/src/test/resources/edq-test.properties new file mode 100644 index 0000000000..8a041c7407 --- /dev/null +++ b/edqs/src/test/resources/edq-test.properties @@ -0,0 +1,2 @@ +zk.enabled=false +service.type=edqs diff --git a/pom.xml b/pom.xml index e70a57fbbf..42747bae46 100755 --- a/pom.xml +++ b/pom.xml @@ -165,6 +165,8 @@ 1.6.1 2.19.0 9.2.0 + 1.1.10.5 + 9.4.0 @@ -172,6 +174,7 @@ common rule-engine dao + edqs transport ui-ngx tools @@ -1031,6 +1034,11 @@ coap-server ${project.version} + + org.thingsboard.common + edqs + ${project.version} + org.thingsboard.common.script script-api @@ -2279,6 +2287,16 @@ metadata-extractor ${drewnoakes-metadata-extractor.version} + + org.xerial.snappy + snappy-java + ${snappy.version} + + + org.rocksdb + rocksdbjni + ${rocksdbjni.version} + From 3d42a4ca04af2c21a89fc7fa11a8f9fd7adb970d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 27 Jan 2025 17:57:39 +0200 Subject: [PATCH 100/438] Actor system implementation draft --- .../CalculatedFieldEntityMessageProcessor.java | 8 ++++---- .../cf/DefaultCalculatedFieldExecutionService.java | 6 ++++-- .../service/cf/ctx/state/CalculatedFieldState.java | 1 + .../org/thingsboard/server/common/util/ProtoUtils.java | 10 ++++++---- .../sql/cf/DefaultNativeCalculatedFieldRepository.java | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 6bb6256563..533eeb5e31 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -78,6 +78,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.ctx = ctx; } + public void process(CalculatedFieldStateRestoreMsg msg) { + states.put(msg.getId().cfId(), msg.getState()); + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -192,8 +196,4 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } - - public void process(CalculatedFieldStateRestoreMsg msg) { - states.put(msg.getId().cfId(), msg.getState()); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index bb8ba8bb63..3e37366da3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -837,8 +837,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + if(calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } } return telemetryMsg; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..f261e586a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,5 +44,6 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 6a40f13688..9bbce7c522 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -644,11 +644,13 @@ public class ProtoUtils { } public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { - return TransportProtos.TsKvProto.newBuilder() + var builder = TransportProtos.TsKvProto.newBuilder() .setTs(tsKvEntry.getTs()) - .setKv(toKeyValueProto(tsKvEntry)) - .setVersion(tsKvEntry.getVersion()) - .build(); + .setKv(toKeyValueProto(tsKvEntry)); + if (tsKvEntry.getVersion() != null) { + builder.setVersion(tsKvEntry.getVersion()); + } + return builder.build(); } public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index bb88982e3d..677234dc20 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -77,7 +77,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF String name = (String) row.get("name"); int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); - long version = (long) row.get("version"); + long version = row.get("version") != null ? (long) row.get("version") : 0; Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); From d54cb300d42757b929bbe94936e6d90b1f5db874 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 28 Jan 2025 08:43:29 +0200 Subject: [PATCH 101/438] added new endpoint --- .../controller/CalculatedFieldController.java | 30 +++++++++++++++++++ .../controller/ControllerConstants.java | 1 + 2 files changed, 31 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 4bf243034b..efe25e3883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; @@ -24,17 +25,26 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -81,6 +91,26 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } + @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", + notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + ) + @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @ResponseBody + public PageData getCalculatedFields( + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) + @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", notes = "Deletes the calculated field. Referencing non-existing Calculated Field Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index c7a59cfd83..29625941bd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -96,6 +96,7 @@ public class ControllerConstants { protected static final String EDGE_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the edge name."; protected static final String EVENT_TEXT_SEARCH_DESCRIPTION = "The value is not used in searching."; protected static final String AUDIT_LOG_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on one of the next properties: entityType, entityName, userName, actionType, actionStatus."; + protected static final String CF_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the calculated field name."; protected static final String SORT_PROPERTY_DESCRIPTION = "Property of entity to sort by"; protected static final String SORT_ORDER_DESCRIPTION = "Sort order. ASC (ASCENDING) or DESC (DESCENDING)"; From 5e16db275cb628bf57d3afea6c2df78bcba86fd3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 29 Jan 2025 14:44:28 +0200 Subject: [PATCH 102/438] state and argument refactoring --- ...efaultCalculatedFieldExecutionService.java | 11 +- .../service/cf/ctx/state/ArgumentEntry.java | 2 +- .../ctx/state/BaseCalculatedFieldState.java | 38 +-- .../cf/ctx/state/CalculatedFieldCtx.java | 5 +- .../cf/ctx/state/CalculatedFieldState.java | 1 - .../ctx/state/ScriptCalculatedFieldState.java | 9 +- .../ctx/state/SimpleCalculatedFieldState.java | 26 +- .../ctx/state/SingleValueArgumentEntry.java | 26 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 48 ++-- .../state/ScriptCalculatedFieldStateTest.java | 238 ++++++++++++++++++ .../state/SimpleCalculatedFieldStateTest.java | 213 ++++++++++++++++ .../state/SingleValueArgumentEntryTest.java | 71 ++++++ .../ctx/state/TsRollingArgumentEntryTest.java | 93 +++++++ 13 files changed, 725 insertions(+), 56 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java create mode 100644 application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..6f19090b6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.id.AssetId; @@ -248,7 +247,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas argFutures.put(entry.getKey(), argValueFuture); } return Futures.whenAllComplete(argFutures.values()).call(() -> { - var result = createStateByType(ctx.getCfType()); + var result = createStateByType(ctx); result.updateState(argFutures.entrySet().stream() .collect(Collectors.toMap( Entry::getKey, // Keep the key as is @@ -798,10 +797,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return payload; } - private CalculatedFieldState createStateByType(CalculatedFieldType calculatedFieldType) { - return switch (calculatedFieldType) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); + private CalculatedFieldState createStateByType(CalculatedFieldCtx ctx) { + return switch (ctx.getCfType()) { + case SIMPLE -> new SimpleCalculatedFieldState(ctx.getArgNames()); + case SCRIPT -> new ScriptCalculatedFieldState(ctx.getArgNames()); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java index b261840bfd..be471a8ad9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java @@ -39,7 +39,7 @@ public interface ArgumentEntry { Object getValue(); - boolean hasUpdatedValue(ArgumentEntry entry); + boolean updateEntry(ArgumentEntry entry); static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { return new SingleValueArgumentEntry(kvEntry); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index ca243e25b5..d623043cee 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -16,14 +16,21 @@ package org.thingsboard.server.service.cf.ctx.state; import java.util.HashMap; +import java.util.List; import java.util.Map; public abstract class BaseCalculatedFieldState implements CalculatedFieldState { + protected List requiredArguments; protected Map arguments; public BaseCalculatedFieldState() { - arguments = new HashMap<>(); + this.arguments = new HashMap<>(); + } + + public BaseCalculatedFieldState(List requiredArguments) { + this.requiredArguments = requiredArguments; + this.arguments = new HashMap<>(); } @Override @@ -44,22 +51,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) { - if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) { - existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords()); - } else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { - existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue()); - } else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) { -// Long existingVersion = existingSingleValueEntry.getVersion(); -// Long newVersion = singleValueEntry.getVersion(); -// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) { -// arguments.put(key, newEntry.copy()); -// } - arguments.put(key, newEntry.copy()); - } else { - arguments.put(key, newEntry.copy()); - } + if (existingEntry == null) { + validateNewEntry(newEntry); + arguments.put(key, newEntry.copy()); stateUpdated = true; + } else { + stateUpdated = existingEntry.updateEntry(newEntry); } } @@ -68,7 +65,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public boolean isReady() { - //TODO: IM - return true; + return arguments.keySet().containsAll(requiredArguments) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && + !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } + + protected void validateNewEntry(ArgumentEntry newEntry) { + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..b9834f18d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; +import net.objecthunter.exp4j.Expression; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -31,7 +32,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import java.util.ArrayList; import java.util.HashMap; @@ -56,6 +56,7 @@ public class CalculatedFieldCtx { private String expression; private TbelInvokeService tbelInvokeService; private CalculatedFieldScriptEngine calculatedFieldScriptEngine; + private ThreadLocal customExpression; public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { this.cfId = calculatedField.getId(); @@ -86,6 +87,8 @@ public class CalculatedFieldCtx { this.tbelInvokeService = tbelInvokeService; if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } else { + this.customExpression = new ThreadLocal<>(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index f261e586a4..4b7918cc03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -44,6 +44,5 @@ public interface CalculatedFieldState { ListenableFuture performCalculation(CalculatedFieldCtx ctx); - @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 0421055fef..290fd95370 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -32,6 +33,10 @@ import java.util.TreeMap; @Slf4j public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { + public ScriptCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; @@ -40,9 +45,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { - if (argumentEntry instanceof TsRollingArgumentEntry) { + if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index e16d310b3e..59b3009bb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -24,21 +24,32 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; -import java.util.HashMap; +import java.util.List; import java.util.Map; @Data public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { + public SimpleCalculatedFieldState(List requiredArguments) { + super(requiredArguments); + } + @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + if (newEntry instanceof TsRollingArgumentEntry) { + throw new IllegalArgumentException("Rolling argument entry is not supported for simple calculated fields."); + } + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { String expression = ctx.getExpression(); - ThreadLocal customExpression = new ThreadLocal<>(); + ThreadLocal customExpression = ctx.getCustomExpression(); var expr = customExpression.get(); if (expr == null) { expr = new ExpressionBuilder(expression) @@ -47,9 +58,14 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { .build(); customExpression.set(expr); } - Map variables = new HashMap<>(); - this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString()))); - expr.setVariables(variables); + + for (Map.Entry entry : this.arguments.entrySet()) { + try { + expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); + } + } double expressionResult = expr.evaluate(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index d5f7b1aee6..5117fcab0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -21,7 +21,6 @@ import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.util.KvProtoUtil; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -35,7 +34,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { private long ts; private Object value; - private Long version; public SingleValueArgumentEntry(TsKvProto entry) { @@ -79,14 +77,28 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return value; } - @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return this.ts != ((SingleValueArgumentEntry) entry).getTs(); - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); } + @Override + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + if (singleValueEntry.getTs() == this.ts) { + return false; + } + + Long newVersion = singleValueEntry.getVersion(); + if (newVersion == null || this.version == null || newVersion > this.version) { + this.ts = singleValueEntry.getTs(); + this.value = singleValueEntry.getValue(); + this.version = newVersion; + return true; + } + } else { + throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); + } + return false; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 1118e3af13..b86a51ca03 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -62,31 +62,49 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean hasUpdatedValue(ArgumentEntry entry) { - return entry instanceof SingleValueArgumentEntry ? - !tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) : - !tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet()); + public ArgumentEntry copy() { + return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); } @Override - public ArgumentEntry copy() { - return new TsRollingArgumentEntry(new TreeMap<>(tsRecords)); + public boolean updateEntry(ArgumentEntry entry) { + if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { + return updateTsRollingEntry(tsRollingEntry); + } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { + return updateSingleValueEntry(singleValueEntry); + } else { + throw new IllegalArgumentException("Unsupported argument entry type for rolling argument entry: " + entry.getType()); + } + } + + private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { + boolean updated = false; + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); + } + return updated; } - public void addTsRecord(Long key, Object value) { + private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + } + + private boolean addTsRecordIfAbsent(Long ts, Object value) { + if (!tsRecords.containsKey(ts)) { + addTsRecord(ts, value); + return true; + } + return false; + } + + private void addTsRecord(Long ts, Object value) { if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(key, value); + tsRecords.put(ts, value); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } } else { - log.warn("Argument type 'TS_ROLLING' only supports numeric values."); - } - } - - public void addAllTsRecords(Map newRecords) { - for (Map.Entry entry : newRecords.entrySet()) { - addTsRecord(entry.getKey(), entry.getValue()); + throw new IllegalArgumentException("Argument type " + getType() + " only supports numeric values."); } } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java new file mode 100644 index 0000000000..cbaea6575c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; +import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(classes = DefaultTbelInvokeService.class) +public class ScriptCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); + + private final long ts = System.currentTimeMillis(); + + private ScriptCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @Autowired + private TbelInvokeService tbelInvokeService; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + state = new ScriptCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SCRIPT); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of("assetHumidity", assetHumidityArgEntry)); + + Map newArgs = Map.of("deviceTemperature", deviceTemperatureArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", assetHumidityArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + Map newArgs = Map.of("assetHumidity", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "assetHumidity", newArgEntry, + "deviceTemperature", deviceTemperatureArgEntry + ) + ); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4);// will not be used for calculation + values.put(ts - 45000, 2);// will not be used for calculation + values.put(ts - 20, 0); + + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000);// will not be used + values.put(ts - 18, 0); + values.put(ts - 16, 0); + values.put(ts - 14, 0); + values.put(ts - 12, 0); + values.put(ts - 10, 0); + argumentEntry.setTsRecords(values); + + state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + + assertThat(state.isReady()).isFalse(); + } + + private TsRollingArgumentEntry createRollingArgEntry() { + TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); + long ts = System.currentTimeMillis(); + + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + argumentEntry.setTsRecords(values); + return argumentEntry; + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(ASSET_ID); + calculatedField.setType(CalculatedFieldType.SCRIPT); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(DEVICE_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); + argument1.setRefEntityKey(refEntityKey1); + argument1.setLimit(5); + argument1.setTimeWindow(30000); + + Argument argument2 = new Argument(); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); + + config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + + Output output = new Output(); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java new file mode 100644 index 0000000000..58a981824c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -0,0 +1,213 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.cf.CalculatedFieldResult; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutionException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SimpleCalculatedFieldStateTest { + + private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); + private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); + private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); + + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + + private SimpleCalculatedFieldState state; + private CalculatedFieldCtx ctx; + + @BeforeEach + void setUp() { + ctx = new CalculatedFieldCtx(getCalculatedField(), null); + state = new SimpleCalculatedFieldState(ctx.getArgNames()); + } + + @Test + void testType() { + assertThat(state.getType()).isEqualTo(CalculatedFieldType.SIMPLE); + } + + @Test + void testUpdateState() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", key3ArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf( + Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + ) + ); + } + + @Test + void testUpdateStateWhenUpdateExistingEntry() { + state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); + + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + Map newArgs = Map.of("key1", newArgEntry); + boolean stateUpdated = state.updateState(newArgs); + + assertThat(stateUpdated).isTrue(); + assertThat(state.getArguments()).containsExactlyInAnyOrderEntriesOf(Map.of("key1", newArgEntry)); + } + + @Test + void testUpdateStateWhenRollingEntryPassed() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + + Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + } + + @Test + void testPerformCalculation() throws ExecutionException, InterruptedException { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + CalculatedFieldResult result = state.performCalculation(ctx).get(); + + assertThat(result).isNotNull(); + Output output = getCalculatedFieldConfig().getOutput(); + assertThat(result.getType()).isEqualTo(output.getType()); + assertThat(result.getScope()).isEqualTo(output.getScope()); + assertThat(result.getResultMap()).isEqualTo(Map.of("output", 49.0)); + } + + @Test + void testPerformCalculationWhenPassedNotNumber() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key3", key3ArgEntry + )); + + assertThatThrownBy(() -> state.performCalculation(ctx)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument 'key2' is not a number."); + } + + @Test + void testIsReadyWhenNotAllArgPresent() { + assertThat(state.isReady()).isFalse(); + } + + @Test + void testIsReadyWhenAllArgPresent() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry, + "key3", key3ArgEntry + )); + + assertThat(state.isReady()).isTrue(); + } + + @Test + void testIsReadyWhenEmptyEntryPresents() { + state.arguments = new HashMap<>(Map.of( + "key1", key1ArgEntry, + "key2", key2ArgEntry + )); + state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + + assertThat(state.isReady()).isFalse(); + } + + private CalculatedField getCalculatedField() { + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setTenantId(TENANT_ID); + calculatedField.setEntityId(DEVICE_ID); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("Test Calculated Field"); + calculatedField.setConfigurationVersion(1); + calculatedField.setConfiguration(getCalculatedFieldConfig()); + calculatedField.setVersion(1L); + return calculatedField; + } + + private CalculatedFieldConfiguration getCalculatedFieldConfig() { + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + argument1.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temp1", ArgumentType.TS_LATEST, null); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("temp2", ArgumentType.ATTRIBUTE, null); + argument2.setRefEntityKey(refEntityKey2); + + Argument argument3 = new Argument(); + argument3.setRefEntityId(ASSET_ID); + ReferencedEntityKey refEntityKey3 = new ReferencedEntityKey("temp3", ArgumentType.TS_LATEST, null); + argument3.setRefEntityKey(refEntityKey3); + + config.setArguments(Map.of("key1", argument1, "key2", argument2, "key3", argument3)); + + config.setExpression("key1 + key2 + key3"); + + Output output = new Output(); + output.setName("output"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + return config; + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java new file mode 100644 index 0000000000..285da0b423 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class SingleValueArgumentEntryTest { + + private SingleValueArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + entry = new SingleValueArgumentEntry(ts, 11, 363L); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.SINGLE_VALUE); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWithThaSameTs() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + } + + @Test + void testUpdateEntryWhenNewVersionIsNull() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.getVersion()).isNull(); + } + + @Test + void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.getVersion()).isEqualTo(369L); + } + + @Test + void testUpdateEntryWhenNewVersionIsLessThanCurrent() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java new file mode 100644 index 0000000000..b08c5f2a58 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.TreeMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TsRollingArgumentEntryTest { + + private TsRollingArgumentEntry entry; + + private final long ts = System.currentTimeMillis(); + + @BeforeEach + void setUp() { + TreeMap values = new TreeMap<>(); + values.put(ts - 40, 10); + values.put(ts - 30, 12); + values.put(ts - 20, 17); + + entry = new TsRollingArgumentEntry(values); + } + + @Test + void testArgumentEntryType() { + assertThat(entry.getType()).isEqualTo(ArgumentEntryType.TS_ROLLING); + } + + @Test + void testUpdateEntryWhenSingleValueEntryPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(4); + assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + } + + @Test + void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + + assertThat(entry.updateEntry(newEntry)).isFalse(); + } + + @Test + void testUpdateEntryWhenRollingEntryPassed() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 16); + values.put(ts - 10, 7); + values.put(ts - 5, 1); + newEntry.setTsRecords(values); + + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(5); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 40, 10, + ts - 30, 12, + ts - 20, 17, + ts - 10, 7, + ts - 5, 1 + )); + } + + @Test + void testUpdateEntryWhenValueIsNotNumber() { + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + + assertThatThrownBy(() -> entry.updateEntry(newEntry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + } + +} \ No newline at end of file From 5aabbd0f1ee683d99dffcde7325f0eb7253dbd05 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 29 Jan 2025 15:37:36 +0200 Subject: [PATCH 103/438] Link dispatch implementation --- .../CalculatedFieldEntityActor.java | 5 +- ...CalculatedFieldEntityMessageProcessor.java | 59 ++++++- .../CalculatedFieldManagerActor.java | 2 + ...alculatedFieldManagerMessageProcessor.java | 125 +++++++++++++- ...tityCalculatedFieldLinkedTelemetryMsg.java | 42 +++++ .../calculatedField/MultipleTbCallback.java | 15 +- .../cf/CalculatedFieldExecutionService.java | 6 +- ...efaultCalculatedFieldExecutionService.java | 159 ++++++++++-------- .../cf/ctx/state/CalculatedFieldCtx.java | 11 ++ ...faultTbCalculatedFieldConsumerService.java | 7 +- .../queue/DefaultTbClusterService.java | 13 ++ .../server/service/queue/TbPackCallback.java | 3 + .../server/cluster/TbClusterService.java | 3 + .../server/common/msg/queue/TbCallback.java | 8 + common/proto/src/main/proto/queue.proto | 3 +- 15 files changed, 365 insertions(+), 96 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index 43f057ad01..a5461c6152 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -43,7 +43,7 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { log.debug("[{}][{}] CF entity actor started.", processor.tenantId, processor.entityId); } catch (Exception e) { log.warn("[{}][{}] Unknown failure", processor.tenantId, processor.entityId, e); - throw new TbActorException("Failed to initialize device actor", e); + throw new TbActorException("Failed to initialize CF entity actor", e); } } @@ -56,6 +56,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; + case CF_LINKED_TELEMETRY_MSG: + processor.process((EntityCalculatedFieldLinkedTelemetryMsg) msg); + break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 533eeb5e31..cea3cce791 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,15 +18,19 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -40,6 +44,8 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -96,7 +102,25 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } } - private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Set cfIds, List cfIdList, MultipleTbCallback callback) { + public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + var proto = msg.getProto(); + var ctx = msg.getCtx(); + var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); + List cfIds = getCalculatedFieldIds(proto); + if (cfIds.contains(ctx.getCfId())) { + callback.onSuccess(CALLBACKS_PER_CF); + } else { + if (proto.getTsDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + } else if (proto.getAttrDataCount() > 0) { + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } + } + + private void process(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, Collection cfIds, List cfIdList, MultipleTbCallback callback) { if (cfIds.contains(ctx.getCfId())) { callback.onSuccess(CALLBACKS_PER_CF); } else { @@ -120,8 +144,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); } + @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) throws InterruptedException, ExecutionException, TimeoutException { + Map newArgValues) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } @@ -159,8 +184,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { + return mapToArguments(ctx.getMainEntityArguments(), data); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, data); + } + + private static Map mapToArguments(Map argNames, List data) { + if (argNames.isEmpty()) { + return Collections.emptyMap(); + } Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (TsKvProto item : data) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKv().getKey(), ArgumentType.TS_LATEST, null); String argName = argNames.get(key); @@ -177,8 +216,19 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } private Map mapToArguments(CalculatedFieldCtx ctx, AttributeScopeProto scope, List attrDataList) { + return mapToArguments(ctx.getMainEntityArguments(), scope, attrDataList); + } + + private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { + var argNames = ctx.getLinkedEntityArguments().get(entityId); + if(argNames.isEmpty()) { + return Collections.emptyMap(); + } + return mapToArguments(argNames, scope, attrDataList); + } + + private static Map mapToArguments(Map argNames, AttributeScopeProto scope, List attrDataList) { Map arguments = new HashMap<>(); - var argNames = ctx.getMainEntityArguments(); for (AttributeValueProto item : attrDataList) { ReferencedEntityKey key = new ReferencedEntityKey(item.getKey(), ArgumentType.ATTRIBUTE, AttributeScope.valueOf(scope.name())); String argName = argNames.get(key); @@ -196,4 +246,5 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } return cfIds; } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 87292d3206..1c198c660d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -73,6 +73,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); break; case CF_LINKED_TELEMETRY_MSG: + processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); + break; case CF_ENTITY_UPDATE_MSG: // processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bc0589e95..4bce7cb322 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,29 +16,52 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -50,18 +73,22 @@ import java.util.concurrent.CopyOnWriteArrayList; @Slf4j public class CalculatedFieldManagerMessageProcessor extends AbstractContextAwareMsgProcessor { - private final Map calculatedFields = new HashMap<>(); + private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); + private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); + private final CalculatedFieldExecutionService cfService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; protected TbActorCtx ctx; final TenantId tenantId; + private final static int initFetchPackSize = 1024; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); + this.cfService = systemContext.getCalculatedFieldExecutionService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -73,11 +100,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - calculatedFields.put(cf.getId(), cf); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()) - .add(new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService())); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -99,14 +126,70 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); - var proto = msg.getProto(); + // 2 = 1 for CF processing + 1 for links processing + MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); - //TODO: Transfer only 'part' of the original callback. - getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, msg.getCallback())); + if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); + } else { + callback.onSuccess(); + } // process all links (if any); - var links = getCalculatedFieldLinksByEntityId(entityId); + List linkedCalculatedFields = filterCalculatedFieldLinks(msg); + var linksSize = linkedCalculatedFields.size(); + if (linksSize > 0) { + cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + } else { + callback.onSuccess(); + } + } + + public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { + EntityId sourceEntityId = msg.getEntityId(); + var proto = msg.getProto(); + var linksList = proto.getLinksList(); + for (var linkProto : linksList) { + var link = toCalculatedFieldEntityCtxId(linkProto); + var targetEntityId = link.entityId(); + var targetEntityType = targetEntityId.getEntityType(); + var cf = calculatedFields.get(link.cfId()); + if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { + // iterate over all entities that belong to profile and push the message for corresponding CF + var entityIds = getEntitiesByProfile(targetEntityId); + if (!entityIds.isEmpty()) { + MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + } else { + msg.getCallback().onSuccess(); + } + } else { + // push the message to specific entity; + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); + getOrCreateActor(targetEntityId).tell(newMsg); + } + } + } + + private CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId(CalculatedFieldEntityCtxIdProto ctxIdProto) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private List filterCalculatedFieldLinks(CalculatedFieldTelemetryMsg msg) { + EntityId entityId = msg.getEntityId(); + var proto = msg.getProto(); + List result = new ArrayList<>(); + for (var link : getCalculatedFieldLinksByEntityId(entityId)) { + CalculatedFieldCtx ctx = calculatedFields.get(link.getCalculatedFieldId()); + if (ctx.linkMatches(entityId, proto)) { + result.add(ctx.toCalculatedFieldEntityCtxId()); + } + } + return result; } private List getCalculatedFieldsByEntityId(EntityId entityId) { @@ -131,6 +214,30 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } + private Set getEntitiesByProfile(EntityId entityProfileId) { + Set entities = profileEntities.get(entityProfileId); + if (entities == null) { + entities = switch (entityProfileId.getEntityType()) { + case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set assetIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getAssetService().findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); + return assetIds; + }); + case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { + Set deviceIds = new HashSet<>(); + (new PageDataIterable<>(pageLink -> + systemContext.getDeviceService().findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); + return deviceIds; + }); + default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); + }; + } + log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); + return entities; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); @@ -139,7 +246,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware }; } - protected TbActorRef getOrCreateActor(EntityId entityId) { + private TbActorRef getOrCreateActor(EntityId entityId) { return ctx.getOrCreateChildActor(new TbCalculatedFieldEntityActorId(entityId), () -> DefaultActorService.CF_ENTITY_DISPATCHER_NAME, () -> new CalculatedFieldEntityActorCreator(systemContext, tenantId, entityId), diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java new file mode 100644 index 0000000000..47b91cbe65 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityCalculatedFieldLinkedTelemetryMsg.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityCalculatedFieldLinkedTelemetryMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final CalculatedFieldTelemetryMsgProto proto; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_LINKED_TELEMETRY_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java index 18e700f38c..312cf72bed 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/MultipleTbCallback.java @@ -15,35 +15,42 @@ */ package org.thingsboard.server.actors.calculatedField; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; +@Slf4j public class MultipleTbCallback implements TbCallback { - + @Getter + private final UUID id; private final AtomicInteger counter; private final TbCallback callback; public MultipleTbCallback(int count, TbCallback callback) { + id = UUID.randomUUID(); this.counter = new AtomicInteger(count); this.callback = callback; } @Override public void onSuccess() { - if (counter.decrementAndGet() <= 0) { - callback.onSuccess(); - } + onSuccess(1); } public void onSuccess(int number) { + log.trace("[{}][{}] onSuccess({})", id, callback.getId(), number); if (counter.addAndGet(-number) <= 0) { + log.trace("[{}][{}] Done.", id, callback.getId()); callback.onSuccess(); } } @Override public void onFailure(Throwable t) { + log.warn("[{}][{}] onFailure.", id, callback.getId()); callback.onFailure(t); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 806c224608..d8daae2e64 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -53,6 +54,10 @@ public interface CalculatedFieldExecutionService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); + void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); + + void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); + // void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); /* ===================================================== */ @@ -65,6 +70,5 @@ public interface CalculatedFieldExecutionService { void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f333eecbd8..19561b73b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -34,6 +34,8 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; +import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -83,6 +85,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -636,60 +639,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } -// private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { -// CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); -// Map argumentsMap = new HashMap<>(argumentValues); -// -// CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); -// -// states.compute(entityCtxId, (ctxId, ctx) -> { -// CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); -// -// CompletableFuture updateFuture = new CompletableFuture<>(); -// -// Consumer performUpdateState = (state) -> { -// if (state.updateState(argumentsMap)) { -// calculatedFieldEntityCtx.setState(state); -// stateService.persistState(entityCtxId, calculatedFieldEntityCtx); -// Map arguments = state.getArguments(); -// boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && -// !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); -// if (allArgsPresent) { -// performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); -// } -// log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); -// } -// updateFuture.complete(null); -// }; -// -// CalculatedFieldState state = calculatedFieldEntityCtx.getState(); -// -// boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); -// boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() -// .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); -// -// if (!allKeysPresent || requiresTsRollingUpdate) { -// Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() -// .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) -// .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); -// -// fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) -// .addListener(() -> performUpdateState.accept(state), -// calculatedFieldCallbackExecutor); -// } else { -// performUpdateState.accept(state); -// } -// -// try { -// updateFuture.join(); -// } catch (Exception e) { -// log.trace("Failed to update state for ctxId [{}].", ctxId, e); -// throw new RuntimeException("Failed to update or initialize state.", e); -// } -// -// return calculatedFieldEntityCtx; -// }); -// } @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { @@ -713,9 +662,74 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }); } catch (Exception e) { log.warn("[{}][{}] Failed to push message to rule engine. CalculatedFieldResult: {}", tenantId, entityId, calculatedFieldResult, e); + callback.onFailure(e); } } + @Override + public void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback) { + Map> unicasts = new HashMap<>(); + List broadcasts = new ArrayList<>(); + for (CalculatedFieldEntityCtxId link : linkedCalculatedFields) { + var linkEntityId = link.entityId(); + var linkEntityType = linkEntityId.getEntityType(); + // Let's assume number of entities in profile is N, and number of partitions is P. If N > P, we save by broadcasting to all partitions. Usually N >> P. + boolean broadcast = EntityType.DEVICE_PROFILE.equals(linkEntityType) || EntityType.ASSET_PROFILE.equals(linkEntityType); + if (broadcast) { + broadcasts.add(link); + } else { + TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); + } + } + MultipleTbCallback linkCallback = new MultipleTbCallback(2, callback); + if (!broadcasts.isEmpty()) { + broadcast(broadcasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + if (!unicasts.isEmpty()) { + unicast(unicasts, msg, linkCallback); + } else { + linkCallback.onSuccess(); + } + } + + private void unicast(Map> unicasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(new MultipleTbCallback(unicasts.size(), mainCallback)); + unicasts.forEach((topicPartitionInfo, ctxIds) -> { + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), ctxIds); + clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), + ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + }); + } + + private void broadcast(List broadcasts, CalculatedFieldTelemetryMsg msg, MultipleTbCallback mainCallback) { + TbQueueCallback callback = new TbCallbackWrapper(mainCallback); + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(msg.getProto(), broadcasts); + clusterService.broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), callback); + } + + private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { + TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + builder.setMsg(telemetryProto); + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + return builder.build(); + } + + //TODO: IM: move to utils; + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -868,25 +882,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); - builder.setMsg(telemetryProto); - for (CalculatedFieldEntityCtxId link : links) { - builder.addLinks(toProto(link)); - } - return builder.build(); - } - - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return TransportProtos.CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -948,4 +943,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private static class TbCallbackWrapper implements TbQueueCallback { + private final TbCallback callback; + + public TbCallbackWrapper(TbCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 1a0a16254a..37e402fbc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -31,7 +31,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; import java.util.HashMap; @@ -143,4 +145,13 @@ public class CalculatedFieldCtx { } return false; } + + public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { + //TODO: IM - implement + return true; + } + + public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { + return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0ccecbbeca..0fcc04bd5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -223,15 +223,16 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); if (toCfNotification.hasComponentLifecycle()) { - // from upstream (maybe removed since we dont need to init state for each partition) + // from upstream (maybe removed since we don't need to init state for each partition) forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); } else if (toCfNotification.hasEntityUpdateMsg()) { processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we dont need to update state for each partition) + // from upstream (maybe removed since we don't need to update state for each partition) forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + } else if (toCfNotification.hasLinkedTelemetryMsg()) { + forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } - callback.onSuccess(); } private void forwardToActorSystem(CalculatedFieldTelemetryMsgProto msg, TbCallback callback) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b1a4b1859e..2783e826fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -194,6 +194,19 @@ public class DefaultTbClusterService implements TbClusterService { } } + @Override + public void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg toCfMsg, TbQueueCallback callback) { + UUID msgId = UUID.randomUUID(); + TbQueueProducer> toCfProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); + Set tbReServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); + for (String serviceId : tbReServices) { + TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); + toCfProducer.send(tpi, new TbProtoQueueMsg<>(msgId, toCfMsg), null); + toRuleEngineNfs.incrementAndGet(); + } + callback.onSuccess(null); // TODO: refactor to be fair, similar to multi-value callback; + } + @Override public void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, TenantId.SYS_TENANT_ID, tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java index f9d15353c7..3402a357d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbPackCallback.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.queue; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -23,9 +24,11 @@ import java.util.UUID; @Slf4j public class TbPackCallback implements TbCallback { private final TbPackProcessingContext ctx; + @Getter private final UUID id; public TbPackCallback(UUID id, TbPackProcessingContext ctx) { + log.trace("[{}] CALLBACK CREATED", id); this.id = id; this.ctx = ctx; } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 269db0a73a..588b135891 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.RestApiCallResponseMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; @@ -62,6 +63,8 @@ public interface TbClusterService extends TbQueueClusterService { void broadcastToCore(ToCoreNotificationMsg msg); + void broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg build, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java index a2d264a4ff..f8204fb0ae 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/TbCallback.java @@ -15,6 +15,10 @@ */ package org.thingsboard.server.common.msg.queue; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.UUID; + public interface TbCallback { TbCallback EMPTY = new TbCallback() { @@ -30,6 +34,10 @@ public interface TbCallback { } }; + default UUID getId(){ + return EntityId.NULL_UUID; + } + void onSuccess(); void onFailure(Throwable t); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 9e3e842cd9..69b912120e 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -803,7 +803,7 @@ message CalculatedFieldTelemetryMsgProto { message CalculatedFieldLinkedTelemetryMsgProto { CalculatedFieldTelemetryMsgProto msg = 1; - repeated CalculatedFieldEntityCtxIdProto links = 2; + repeated CalculatedFieldEntityCtxIdProto links = 2; } message CalculatedFieldEntityCtxIdProto { @@ -1639,6 +1639,7 @@ message ToCalculatedFieldMsg { message ToCalculatedFieldNotificationMsg { ComponentLifecycleMsgProto componentLifecycle = 1; CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From a4aa2444acf62026d11f4601076885ba2ccc213c Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 30 Jan 2025 11:01:45 +0200 Subject: [PATCH 104/438] added new endpoint --- .../controller/CalculatedFieldController.java | 35 ++++++++++--------- .../cf/DefaultTbCalculatedFieldService.java | 9 +++++ .../entitiy/cf/TbCalculatedFieldService.java | 5 +++ .../server/dao/cf/CalculatedFieldService.java | 2 ++ .../dao/cf/BaseCalculatedFieldService.java | 8 +++++ .../server/dao/cf/CalculatedFieldDao.java | 2 ++ .../dao/sql/cf/CalculatedFieldRepository.java | 4 +++ ...efaultNativeCalculatedFieldRepository.java | 2 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 ++++ 9 files changed, 56 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index efe25e3883..124dd8d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -32,6 +32,8 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; @@ -40,7 +42,8 @@ import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; @@ -91,25 +94,25 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFields)", - notes = "Returns a page of calculated fields. " + PAGE_DATA_PARAMETERS + @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody - public PageData getCalculatedFields( - @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) - @RequestParam int pageSize, - @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page, - @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) - @RequestParam(required = false) String textSearch, - @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) - @RequestParam(required = false) String sortProperty, - @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) - @RequestParam(required = false) String sortOrder) throws ThingsboardException { + public PageData getCalculatedFieldsByEntityId( + @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, + @Parameter(description = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable("entityId") String entityIdStr, + @Parameter(description = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, + @Parameter(description = PAGE_NUMBER_DESCRIPTION, required = true) @RequestParam int page, + @Parameter(description = CF_TEXT_SEARCH_DESCRIPTION) @RequestParam(required = false) String textSearch, + @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"createdTime", "name"})) @RequestParam(required = false) String sortProperty, + @Parameter(description = SORT_ORDER_DESCRIPTION, schema = @Schema(allowableValues = {"ASC", "DESC"})) @RequestParam(required = false) String sortOrder) throws ThingsboardException { PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); - return checkNotNull(calculatedFieldService.findAllCalculatedFields(pageLink)); + checkParameter("entityId", entityIdStr); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityIdStr); + checkEntityId(entityId, Operation.READ_CALCULATED_FIELD); + return checkNotNull(tbCalculatedFieldService.findAllByTenantIdAndEntityId(entityId, getCurrentUser(), pageLink)); } @ApiOperation(value = "Delete Calculated Field (deleteCalculatedField)", diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 2e6e975636..c8c589691b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -74,6 +76,13 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp return calculatedFieldService.findById(user.getTenantId(), calculatedFieldId); } + @Override + public PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink) { + TenantId tenantId = user.getTenantId(); + checkEntityExistence(tenantId, entityId); + return calculatedFieldService.findAllCalculatedFieldsByEntityId(tenantId, entityId, pageLink); + } + @Override @Transactional public void delete(CalculatedField calculatedField, SecurityUser user) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java index 89931b8541..cb7e809059 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/TbCalculatedFieldService.java @@ -18,6 +18,9 @@ package org.thingsboard.server.service.entitiy.cf; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; public interface TbCalculatedFieldService { @@ -26,6 +29,8 @@ public interface TbCalculatedFieldService { CalculatedField findById(CalculatedFieldId calculatedFieldId, SecurityUser user); + PageData findAllByTenantIdAndEntityId(EntityId entityId, SecurityUser user, PageLink pageLink); + void delete(CalculatedField calculatedField, SecurityUser user); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 3a508a5c08..f08d54190f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -44,6 +44,8 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findAllCalculatedFields(PageLink pageLink); + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); int deleteAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 26ed4134cc..d37a4a53c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -118,6 +118,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findAll(pageLink); } + @Override + public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByEntityId(tenantId, entityId, pageLink); + } + @Override public void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { validateId(tenantId, id -> INCORRECT_TENANT_ID + id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 39663d0afc..23a2eae93e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -37,6 +37,8 @@ public interface CalculatedFieldDao extends Dao { PageData findAll(PageLink pageLink); + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); + List removeAllByEntityId(TenantId tenantId, EntityId entityId); boolean existsByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index bed6f2d3a2..c0118d4f02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.cf; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -30,6 +32,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index 677234dc20..fcbf4d4dd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -83,7 +83,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); calculatedField.setCreatedTime(createdTime); - calculatedField.setTenantId(new TenantId(tenantId)); + calculatedField.setTenantId(TenantId.fromUUID(tenantId)); calculatedField.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); calculatedField.setType(type); calculatedField.setName(name); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index cdcffdd440..34bfa27b16 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -71,6 +71,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { + log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override @Transactional public List removeAllByEntityId(TenantId tenantId, EntityId entityId) { From c8db304ce6e067dcd00791054ca58fa6b0ad9123 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 30 Jan 2025 12:43:14 +0200 Subject: [PATCH 105/438] Optimized postgres sync (#3208) * optimized postgres sync * Fix consumer stopping when stopWhenRead enabled * Fix NPE on EDQS repartitioning * fixed EntityServiceTest * Fix EDQS yml props --------- Co-authored-by: ViacheslavKlimov --- .../service/edqs/DefaultEdqsService.java | 1 + .../server/service/edqs/EdqsDataLoader.java | 539 ------------------ .../server/service/edqs/EdqsSyncService.java | 102 +++- .../service/edqs/KafkaEdqsSyncService.java | 2 +- .../src/main/resources/thingsboard.yml | 5 +- .../service/entitiy/EntityServiceTest.java | 267 ++++++++- .../common/data/edqs/fields/FieldsUtil.java | 2 + .../server/edqs/processor/EdqsProcessor.java | 30 +- .../ApiUsageStateQueryProcessor.java | 3 +- .../edqs/state/KafkaEdqsStateService.java | 18 +- .../queue/discovery/HashPartitionService.java | 2 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 16 +- .../common/stats/DefaultStatsFactory.java | 2 +- .../server/dao/attributes/AttributesDao.java | 7 +- .../server/dao/relation/RelationDao.java | 4 - .../sql/attributes/AttributeKvRepository.java | 8 + .../dao/sql/attributes/JpaAttributeDao.java | 10 +- .../query/DefaultEntityQueryRepository.java | 4 + .../dao/sql/relation/JpaRelationDao.java | 6 - .../dao/sql/relation/RelationRepository.java | 13 +- .../CachedRedisSqlTimeseriesLatestDao.java | 4 - .../dao/sqlts/SqlTimeseriesLatestDao.java | 4 - .../sqlts/latest/TsKvLatestRepository.java | 8 + .../CassandraBaseTimeseriesLatestDao.java | 4 - .../dao/timeseries/TimeseriesLatestDao.java | 1 - .../edqs/ThingsboardEdqsApplication.java | 10 +- edqs/src/main/resources/edqs.yml | 7 +- 27 files changed, 426 insertions(+), 653 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 7582d036e2..160d4bb565 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -125,6 +125,7 @@ public class DefaultEdqsService implements EdqsService { executor.submit(() -> { try { EdqsSyncState syncState = getSyncState(); + // FIXME: Slavik smart events check if (edqsSyncService.isSyncNeeded() || syncState == null || syncState.getStatus() != EdqsSyncStatus.FINISHED) { if (hashPartitionService.isSystemPartitionMine(ServiceType.TB_CORE)) { processSystemRequest(ToCoreEdqsRequest.builder() diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java deleted file mode 100644 index 69a3f6108d..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsDataLoader.java +++ /dev/null @@ -1,539 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.edqs; - -import com.fasterxml.jackson.databind.MappingIterator; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.ApiUsageState; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceProfileType; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.asset.AssetProfile; -import org.thingsboard.server.common.data.converter.Converter; -import org.thingsboard.server.common.data.converter.ConverterType; -import org.thingsboard.server.common.data.edge.Edge; -import org.thingsboard.server.common.data.edqs.AttributeKv; -import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.group.EntityGroup; -import org.thingsboard.server.common.data.id.ApiUsageStateId; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.ConverterId; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.EntityGroupId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.IntegrationId; -import org.thingsboard.server.common.data.id.RoleId; -import org.thingsboard.server.common.data.id.RuleChainId; -import org.thingsboard.server.common.data.id.SchedulerEventId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.TenantProfileId; -import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.id.WidgetTypeId; -import org.thingsboard.server.common.data.id.WidgetsBundleId; -import org.thingsboard.server.common.data.integration.Integration; -import org.thingsboard.server.common.data.integration.IntegrationType; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.common.data.role.Role; -import org.thingsboard.server.common.data.role.RoleType; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.scheduler.SchedulerEvent; -import org.thingsboard.server.common.data.widget.WidgetType; -import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.common.msg.edqs.EdqsService; -import org.thingsboard.server.edqs.processor.EdqsConverter; - -import java.io.FileReader; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -import static org.thingsboard.common.util.JacksonUtil.toJsonNode; - - -@RequiredArgsConstructor -@Slf4j -//@Service -public class EdqsDataLoader { - - private final EdqsService edqsService; - private final EdqsConverter edqsConverter; - - public final static TenantId MAIN = TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")); - - private final String folder = "/home/viacheslav/Downloads/schwarz"; - - private ExecutorService executor = Executors.newFixedThreadPool(5, ThingsBoardThreadFactory.forName("edqs-publisher")); - -// @AfterStartUp(order = 100) - public void load() throws Exception { - loadCustomers(); - loadDeviceProfile(); - loadDevices(); - loadAssets(); - loadEdges(); - loadEntityViews(); - loadTenants(); - loadUsers(); - loadDashboards(); - loadRuleChains(); - loadWidgetType(); - loadWidgetBundle(); - loadConverters(); - loadIntegrations(); - loadSchedulerEvents(); - loadRoles(); - loadApiUsageStates(); - loadAssetProfile(); - loadEntityGroups(); - loadRelations(); - - loadAttributes(); - loadTs(); - } - - private void loadCustomers() throws Exception { - load("customer.csv", (values) -> { - Customer customer = new Customer(); - customer.setTitle(values.get("title")); - customer.setId(new CustomerId(UUID.fromString(values.get("id")))); - customer.setCreatedTime(Long.parseLong(values.get("created_time"))); - customer.setTenantId(tenantId(values.get("tenant_id"))); - var parentCustomerId = values.get("parent_customer_id"); - if (StringUtils.isNotEmpty(parentCustomerId)) { - customer.setParentCustomerId(new CustomerId(UUID.fromString(parentCustomerId))); - } - edqsService.onUpdate(customer.getTenantId(), customer.getId(), customer); - }); - } - - private void loadDevices() throws Exception { - load("device.csv", (values) -> { - Device device = new Device(); - device.setType(values.get("type")); - device.setName(values.get("name")); - device.setLabel(values.get("label")); - device.setId(new DeviceId(uuid(values.get("id")))); - device.setCreatedTime(parseLong(values.get("created_time"))); - device.setCustomerId(customerId(values.get("customer_id"))); - device.setTenantId(tenantId(values.get("tenant_id"))); - device.setDeviceProfileId(new DeviceProfileId(uuid(values.get("device_profile_id")))); - device.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(device.getTenantId(), device.getId(), device); - }); - } - - private void loadAssets() throws Exception { - load("asset.csv", (values) -> { - Asset asset = new Asset(); - asset.setType(values.get("type")); - asset.setName(values.get("name")); - asset.setLabel(values.get("label")); - asset.setId(new AssetId(uuid(values.get("id")))); - asset.setCreatedTime(parseLong(values.get("created_time"))); - asset.setCustomerId(customerId(values.get("customer_id"))); - asset.setTenantId(tenantId(values.get("tenant_id"))); - asset.setAssetProfileId(new AssetProfileId(uuid(values.get("asset_profile_id")))); - asset.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(asset.getTenantId(), asset.getId(), asset); - }); - } - - private void loadEdges() throws Exception { - load("edge.csv", (values) -> { - Edge edge = new Edge(); - edge.setId(new EdgeId(uuid(values.get("id")))); - edge.setCreatedTime(parseLong(values.get("created_time"))); - edge.setType(values.get("type")); - edge.setName(values.get("name")); - edge.setLabel(values.get("label")); - edge.setCustomerId(customerId(values.get("customer_id"))); - edge.setTenantId(tenantId(values.get("tenant_id"))); - edge.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(edge.getTenantId(), edge.getId(), edge); - }); - } - - private void loadEntityViews() throws Exception { - load("entity_view.csv", (values) -> { - EntityView entityView = new EntityView(); - entityView.setId(new EntityViewId(uuid(values.get("id")))); - entityView.setCreatedTime(parseLong(values.get("created_time"))); - entityView.setType(values.get("type")); - entityView.setName(values.get("name")); - entityView.setCustomerId(customerId(values.get("customer_id"))); - entityView.setTenantId(tenantId(values.get("tenant_id"))); - entityView.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(entityView.getTenantId(), entityView.getId(), entityView); - }); - } - - private void loadTenants() throws Exception { - load("tenant.csv", (values) -> { - Tenant tenant = new Tenant(); - tenant.setId(new TenantId(uuid(values.get("id")))); - tenant.setCreatedTime(parseLong(values.get("created_time"))); - tenant.setEmail(values.get("email")); - tenant.setTitle(values.get("title")); - tenant.setCountry(values.get("country")); - tenant.setState(values.get("state")); - tenant.setCity(values.get("city")); - tenant.setAddress(values.get("address")); - tenant.setAddress2(values.get("address2")); - tenant.setZip(values.get("zip")); - tenant.setPhone(values.get("phone")); - tenant.setRegion(values.get("region")); - tenant.setTenantProfileId(new TenantProfileId(uuid(values.get("tenant_profile_id")))); - tenant.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - edqsService.onUpdate(MAIN, tenant.getId(), tenant); - }); - } - - private void loadUsers() throws Exception { - load("user.csv", (values) -> { - User user = new User(); - user.setId(new UserId(uuid(values.get("id")))); - user.setCreatedTime(parseLong(values.get("created_time"))); - user.setTenantId(tenantId(values.get("tenant_id"))); - user.setFirstName(values.get("first_name")); - user.setLastName(values.get("last_name")); - user.setEmail(values.get("email")); - user.setPhone(values.get("phone")); - user.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(user.getTenantId(), user.getId(), user); - }); - } - - private void loadDashboards() throws Exception { - load("dashboard.csv", (values) -> { - Dashboard dashboard = new Dashboard(); - dashboard.setId(new DashboardId(uuid(values.get("id")))); - dashboard.setCreatedTime(parseLong(values.get("created_time"))); - dashboard.setTenantId(tenantId(values.get("tenant_id"))); - dashboard.setTitle(values.get("title")); - - edqsService.onUpdate(dashboard.getTenantId(), dashboard.getId(), dashboard); - }); - } - - private void loadEntityGroups() throws Exception { - load("entity_group.csv", (values) -> { - EntityGroup entityGroup = new EntityGroup(); - entityGroup.setId(new EntityGroupId(uuid(values.get("id")))); - entityGroup.setCreatedTime(parseLong(values.get("created_time"))); - entityGroup.setName(values.get("name")); - entityGroup.setOwnerId(entityId(values.get("owner_type"), values.get("owner_id"))); - entityGroup.setType(EntityType.valueOf(values.get("type"))); - edqsService.onUpdate(MAIN, entityGroup.getId(), entityGroup); - }); - } - - private void loadRelations() throws Exception { - load("relation.csv", (values) -> { - EntityRelation entityRelation = new EntityRelation(); - entityRelation.setFrom(entityId(values.get("from_type"), values.get("from_id"))); - entityRelation.setTo(entityId(values.get("to_type"), values.get("to_id"))); - entityRelation.setTypeGroup(RelationTypeGroup.valueOf(values.get("relation_type_group"))); - entityRelation.setType(values.get("relation_type")); - edqsService.onUpdate(MAIN, ObjectType.RELATION, entityRelation); - }); - } - - private void loadRuleChains() throws Exception { - load("rule_chain.csv", (values) -> { - RuleChain ruleChain = new RuleChain(); - ruleChain.setId(new RuleChainId(uuid(values.get("id")))); - ruleChain.setCreatedTime(parseLong(values.get("created_time"))); - ruleChain.setName(values.get("name")); - ruleChain.setTenantId(tenantId(values.get("tenant_id"))); - ruleChain.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(ruleChain.getTenantId(), ruleChain.getId(), ruleChain); - }); - } - - private void loadWidgetType() throws Exception { - load("widget_type.csv", (values) -> { - WidgetType widgetType = new WidgetType(); - widgetType.setId(new WidgetTypeId(uuid(values.get("id")))); - widgetType.setCreatedTime(parseLong(values.get("created_time"))); - widgetType.setName(values.get("name")); - widgetType.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(widgetType.getTenantId(), widgetType.getId(), widgetType); - }); - } - - private void loadWidgetBundle() throws Exception { - load("widgets_bundle.csv", (values) -> { - WidgetsBundle widgetsBundle = new WidgetsBundle(); - widgetsBundle.setId(new WidgetsBundleId(uuid(values.get("id")))); - widgetsBundle.setCreatedTime(parseLong(values.get("created_time"))); - widgetsBundle.setTitle(values.get("title")); - widgetsBundle.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle); - }); - } - - private void loadConverters() throws Exception { - load("converter.csv", (values) -> { - Converter converter = new Converter(); - converter.setId(new ConverterId(uuid(values.get("id")))); - converter.setCreatedTime(parseLong(values.get("created_time"))); - converter.setName(values.get("name")); - converter.setType(ConverterType.valueOf(values.get("type"))); - converter.setTenantId(tenantId(values.get("tenant_id"))); - converter.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); - converter.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(converter.getTenantId(), converter.getId(), converter); - }); - } - - private void loadIntegrations() throws Exception { - load("integration.csv", (values) -> { - Integration integration = new Integration(); - integration.setId(new IntegrationId(uuid(values.get("id")))); - integration.setCreatedTime(parseLong(values.get("created_time"))); - integration.setName(values.get("name")); - integration.setType(IntegrationType.valueOf(values.get("type"))); - integration.setTenantId(tenantId(values.get("tenant_id"))); - integration.setEdgeTemplate(parseBoolean(values.get("is_edge_template"))); - integration.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(integration.getTenantId(), integration.getId(), integration); - }); - } - - private void loadSchedulerEvents() throws Exception { - load("scheduler_event.csv", (values) -> { - SchedulerEvent schedulerEvent = new SchedulerEvent(); - schedulerEvent.setId(new SchedulerEventId(uuid(values.get("id")))); - schedulerEvent.setCreatedTime(parseLong(values.get("created_time"))); - schedulerEvent.setName(values.get("name")); - schedulerEvent.setType(values.get("type")); - schedulerEvent.setTenantId(tenantId(values.get("tenant_id"))); - schedulerEvent.setConfiguration(toJsonNode(values.get("configuration"))); - schedulerEvent.setSchedule(toJsonNode(values.get("schedule"))); - schedulerEvent.setOriginatorId(entityId(values.get("originator_type"), values.get("originator_id"))); - schedulerEvent.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(schedulerEvent.getTenantId(), schedulerEvent.getId(), schedulerEvent); - }); - } - - private void loadRoles() throws Exception { - load("role.csv", (values) -> { - Role role = new Role(); - role.setId(new RoleId(uuid(values.get("id")))); - role.setCreatedTime(parseLong(values.get("created_time"))); - role.setName(values.get("name")); - role.setType(RoleType.valueOf(values.get("type"))); - role.setTenantId(tenantId(values.get("tenant_id"))); - role.setAdditionalInfo(toJsonNode(values.get("additional_info"))); - - edqsService.onUpdate(role.getTenantId(), role.getId(), role); - }); - } - - private void loadApiUsageStates() throws Exception { - load("api_usage_state.csv", (values) -> { - ApiUsageState apiUsageState = new ApiUsageState(); - apiUsageState.setId(new ApiUsageStateId(uuid(values.get("id")))); - apiUsageState.setCreatedTime(parseLong(values.get("created_time"))); - apiUsageState.setEntityId(entityId(values.get("entity_type"), values.get("entity_id"))); - apiUsageState.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(apiUsageState.getTenantId(), apiUsageState.getId(), apiUsageState); - }); - } - - private void loadDeviceProfile() throws Exception { - load("device_profile.csv", (values) -> { - DeviceProfile deviceProfile = new DeviceProfile(); - deviceProfile.setId(new DeviceProfileId(uuid(values.get("id")))); - deviceProfile.setCreatedTime(parseLong(values.get("created_time"))); - deviceProfile.setName(values.get("name")); - deviceProfile.setType(DeviceProfileType.valueOf(values.get("type"))); - deviceProfile.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile); - }); - } - - private void loadAssetProfile() throws Exception { - load("asset_profile.csv", (values) -> { - AssetProfile assetProfile = new AssetProfile(); - assetProfile.setId(new AssetProfileId(uuid(values.get("id")))); - assetProfile.setCreatedTime(parseLong(values.get("created_time"))); - assetProfile.setName(values.get("name")); - assetProfile.setTenantId(tenantId(values.get("tenant_id"))); - - edqsService.onUpdate(assetProfile.getTenantId(), assetProfile.getId(), assetProfile); - }); - } - - private void loadAttributes() throws Exception { - load("attribute.csv", (values) -> { - EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("entity_type"), values.get("entity_id")); - long ts = parseLong(values.get("last_update_ts")); - AttributeScope scope = AttributeScope.valueOf(values.get("attribute_type")); - String key = values.get("attribute_key"); - KvEntry kvEntry; - if (StringUtils.isNotEmpty(values.get("bool_v"))) { - kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); - } else if (StringUtils.isNotEmpty(values.get("str_v"))) { - kvEntry = new StringDataEntry(key, values.get("str_v")); - } else if (StringUtils.isNotEmpty(values.get("long_v"))) { - kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); - } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { - kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); - } else if (StringUtils.isNotEmpty(values.get("json_v"))) { - kvEntry = new JsonDataEntry(key, values.get("json_v")); - } else { - kvEntry = new StringDataEntry(key, ""); - } - AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(ts, kvEntry); - AttributeKv attributeKv = new AttributeKv(entityId, scope, attributeKvEntry, 0); - edqsService.onUpdate(MAIN, ObjectType.ATTRIBUTE_KV, attributeKv); - }); - } - - private void loadTs() throws Exception { - load("ts_kv.csv", (values) -> { - var entityTypeStr = values.get("find_entity_type"); - if (StringUtils.isEmpty(entityTypeStr)) { - return; - } - EntityId entityId = EntityIdFactory.getByTypeAndId(values.get("find_entity_type"), values.get("entity_id")); - long ts = parseLong(values.get("ts")); - String key = values.get("key"); - KvEntry kvEntry; - if (StringUtils.isNotEmpty(values.get("bool_v"))) { - kvEntry = new BooleanDataEntry(key, "t".equals(values.get("bool_v"))); - } else if (StringUtils.isNotEmpty(values.get("str_v"))) { - kvEntry = new StringDataEntry(key, values.get("str_v")); - } else if (StringUtils.isNotEmpty(values.get("long_v"))) { - kvEntry = new LongDataEntry(key, parseLong(values.get("long_v"))); - } else if (StringUtils.isNotEmpty(values.get("dbl_v"))) { - kvEntry = new DoubleDataEntry(key, Double.parseDouble(values.get("dbl_v"))); - } else if (StringUtils.isNotEmpty(values.get("json_v"))) { - kvEntry = new JsonDataEntry(key, values.get("json_v")); - } else { - kvEntry = new StringDataEntry(key, ""); - } - BasicTsKvEntry tsKvEntry = new BasicTsKvEntry(ts, kvEntry); - edqsService.onUpdate(MAIN, ObjectType.LATEST_TS_KV, new LatestTsKv(entityId, tsKvEntry, 0L)); - }); - } - - private void load(String file, Consumer> function) throws Exception { - Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("loader-" + file)).submit(() -> { - try { - long ts = System.currentTimeMillis(); - CsvSchema schema = CsvSchema.emptySchema().withHeader().withColumnSeparator('|'); - CsvMapper mapper = new CsvMapper(); - MappingIterator> it = mapper - .readerFor(Map.class) - .with(schema) - .readValues(new FileReader(folder + "/" + file)); - - int success = 0; - int failure = 0; - while (it.hasNextValue()) { - Map row = it.nextValue(); - try { - function.accept(row); - success++; - if (success % 1000 == 0) { - log.info("Loaded [{}] from [{}]", success, file); - } - } catch (Exception e) { - log.error("Failed to parse str: [{}]", row, e); - failure++; - } - } - log.info("Loaded [{}] from [{}] in {}ms. Failures {}", success, file, (System.currentTimeMillis() - ts), failure); - } catch (Throwable t) { - log.error("Failed to load data from [{}]", file, t); - } - }); - } - - private static TenantId tenantId(String id) { - return TenantId.fromUUID(UUID.fromString(id)); - } - - private static CustomerId customerId(String id) { - var c = new CustomerId(UUID.fromString(id)); - return c.isNullUid() ? null : c; - } - - private static EntityId entityId(String type, String id) { - return EntityIdFactory.getByTypeAndId(type, id); - } - - private static UUID uuid(String id) { - return UUID.fromString(id); - } - - private static long parseLong(String time) { - return Long.parseLong(time); - } - - private static boolean parseBoolean(String value) { - return Boolean.parseBoolean(value); - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 8e12da56e0..bf8e096059 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -32,8 +32,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.attributes.AttributesDao; @@ -41,13 +39,15 @@ import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; import org.thingsboard.server.dao.group.EntityGroupDao; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; +import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.sql.relation.RelationRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -92,11 +92,11 @@ public abstract class EdqsSyncService { @Autowired private KeyDictionaryDao keyDictionaryDao; @Autowired - private RelationDao relationDao; + private RelationRepository relationRepository; @Autowired private EntityGroupDao entityGroupDao; @Autowired - private TimeseriesLatestDao timeseriesLatestDao; + private TsKvLatestRepository tsKvLatestRepository; @Autowired @Lazy private DefaultEdqsService edqsService; @@ -171,7 +171,7 @@ public abstract class EdqsSyncService { private void syncEntityGroups() { log.info("Synchronizing entity groups to EDQS"); long ts = System.currentTimeMillis(); - var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 10000); + var entityGroups = new PageDataIterable<>(entityGroupDao::findAllFields, 30000); for (EntityFields groupFields : entityGroups) { EntityIdInfo entityIdInfo = entityInfoMap.get(groupFields.getOwnerId()); if (entityIdInfo != null) { @@ -187,18 +187,43 @@ public abstract class EdqsSyncService { private void syncRelations() { log.info("Synchronizing relations to EDQS"); long ts = System.currentTimeMillis(); - var relations = new PageDataIterable<>(relationDao::findAll, 10000); - for (EntityRelation relation : relations) { - if (relation.getTypeGroup() == RelationTypeGroup.COMMON || relation.getTypeGroup() == RelationTypeGroup.FROM_ENTITY_GROUP) { - EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFrom().getId()); + UUID lastFromEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastFromEntityType = ""; + String lastRelationTypeGroup = ""; + String lastRelationType = ""; + UUID lastToEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + String lastToEntityType = ""; + + while (true) { + List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, + lastRelationType, lastToEntityId, lastToEntityType, 10000); + if (batch.isEmpty()) { + break; + } + processRelationBatch(batch); + + RelationEntity lastRecord = batch.get(batch.size() - 1); + lastFromEntityId = lastRecord.getFromId(); + lastFromEntityType = lastRecord.getFromType(); + lastRelationTypeGroup = lastRecord.getRelationTypeGroup(); + lastRelationType = lastRecord.getRelationType(); + lastToEntityId = lastRecord.getToId(); + lastToEntityType = lastRecord.getToType(); + } + log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processRelationBatch(List relations) { + for (RelationEntity relation : relations) { + if (RelationTypeGroup.COMMON.name().equals(relation.getRelationTypeGroup()) || (RelationTypeGroup.FROM_ENTITY_GROUP.name().equals(relation.getRelationTypeGroup()))) { + EntityIdInfo entityIdInfo = entityInfoMap.get(relation.getFromId()); if (entityIdInfo != null) { - process(entityIdInfo.tenantId(), RELATION, relation); + process(entityIdInfo.tenantId(), RELATION, relation.toData()); } else { - log.info("Relation from entity not found: " + relation.getFrom()); + log.info("Relation from entity not found: " + relation.getFromId()); } } } - log.info("Finished synchronizing relations to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void loadKeyDictionary() { @@ -214,8 +239,28 @@ public abstract class EdqsSyncService { private void syncAttributes() { log.info("Synchronizing attributes to EDQS"); long ts = System.currentTimeMillis(); - var attributes = new PageDataIterable<>(attributesDao::findAll, 10000); - for (AttributeKvEntity attribute : attributes) { + + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastAttributeType = Integer.MIN_VALUE; + int lastAttributeKey = Integer.MIN_VALUE; + + while (true) { + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000); + if (batch.isEmpty()) { + break; + } + processAttributeBatch(batch); + + AttributeKvEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getId().getEntityId(); + lastAttributeType = lastRecord.getId().getAttributeType(); + lastAttributeKey = lastRecord.getId().getAttributeKey(); + } + log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processAttributeBatch(List batch) { + for (AttributeKvEntity attribute : batch) { attribute.setStrKey(getStrKeyOrFetchFromDb(attribute.getId().getAttributeKey())); UUID entityId = attribute.getId().getEntityId(); EntityIdInfo entityIdInfo = entityInfoMap.get(entityId); @@ -230,13 +275,29 @@ public abstract class EdqsSyncService { attribute.getVersion()); process(entityIdInfo.tenantId(), ATTRIBUTE_KV, attributeKv); } - log.info("Finished synchronizing attributes to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private void syncLatestTimeseries() { log.info("Synchronizing latest timeseries to EDQS"); long ts = System.currentTimeMillis(); - var tsKvLatestEntities = new PageDataIterable<>(pageLink -> timeseriesLatestDao.findAllLatest(pageLink), 10000); + UUID lastEntityId = UUID.fromString("00000000-0000-0000-0000-000000000000"); + int lastKey = Integer.MIN_VALUE; + + while (true) { + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + if (batch.isEmpty()) { + break; + } + processTsKvLatestBatch(batch); + + TsKvLatestEntity lastRecord = batch.get(batch.size() - 1); + lastEntityId = lastRecord.getEntityId(); + lastKey = lastRecord.getKey(); + } + log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); + } + + private void processTsKvLatestBatch(List tsKvLatestEntities) { for (TsKvLatestEntity tsKvLatestEntity : tsKvLatestEntities) { try { String strKey = getStrKeyOrFetchFromDb(tsKvLatestEntity.getKey()); @@ -256,7 +317,6 @@ public abstract class EdqsSyncService { log.error("Failed to sync latest timeseries: {}", tsKvLatestEntity, e); } } - log.info("Finished synchronizing latest timeseries to EDQS in {} ms", (System.currentTimeMillis() - ts)); } private String getStrKeyOrFetchFromDb(int key) { @@ -265,7 +325,9 @@ public abstract class EdqsSyncService { return strKey; } else { strKey = keyDictionaryDao.getKey(key); - keys.putIfAbsent(key, strKey); + if (strKey != null) { + keys.put(key, strKey); + } } return strKey; } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index d51d87ed01..f4e9a02b45 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -40,7 +40,7 @@ public class KafkaEdqsSyncService extends EdqsSyncService { @Override public boolean isSyncNeeded() { - return kafkaAdmin.isTopicEmpty(EdqsQueue.STATE.getTopic()); + return kafkaAdmin.isTopicEmpty(EdqsQueue.EVENTS.getTopic()); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9b182ce3a7..dd0e5948c7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1616,11 +1616,11 @@ queue: # Kafka properties for Edge event topic edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1701,6 +1701,7 @@ queue: local: rocksdb_path: "${TB_EDQS_ROCKSDB_PATH:/tmp/edqs-backup}" partitions: "${TB_EDQS_PARTITIONS:12}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index dc29cec3cc..04bad95be7 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -210,7 +210,6 @@ public class EntityServiceTest extends AbstractControllerTest { countByQueryAndCheck(countQuery, 0); } - @Test public void testCountHierarchicalEntitiesByQuery() throws InterruptedException { List assets = new ArrayList<>(); @@ -463,6 +462,7 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + // fails for sql implementation until we fix the issue with the relation query @Test public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { List buildings = new ArrayList<>(); @@ -1443,18 +1443,29 @@ public class EntityServiceTest extends AbstractControllerTest { String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(customerDevices.get(0).getName()); + // find by customer user with generic permission + MergedUserPermissions mergedGenericPermission = new MergedUserPermissions(Map.of(Resource.DEVICE, Set.of(Operation.READ)), Collections.emptyMap()); + PageData customerResults = findByQueryAndCheck(customerId, mergedGenericPermission, query, 1); + + String cutomerDeviceName = customerResults.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(cutomerDeviceName).isEqualTo(customerDevices.get(0).getName()); + + // find by customer user with group permission + MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(customerDeviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); + PageData result2 = findByQueryAndCheck(customerId, mergedGroupOnlyPermission, query, 1); + + String resultDeviceName2 = result2.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(resultDeviceName2).isEqualTo(customerDevices.get(0).getName()); + // try to find tenant device by customer user SingleEntityFilter tenantDeviceFilter = new SingleEntityFilter(); tenantDeviceFilter.setSingleEntity(tenantDevices.get(0).getId()); EntityDataQuery customerQuery2 = new EntityDataQuery(tenantDeviceFilter, pageLink, entityFields, null, null); - PageData customerResults2 = entityService.findEntityDataByQuery(tenantId, customerId, customerQuery2); - - assertEquals(0, customerResults2.getTotalElements()); + findByQueryAndCheck(customerId, mergedGenericPermission, customerQuery2, 0); // find by tenant user with group permission - PageData results3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); + PageData results3 = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), mergedGroupOnlyPermission, query, 1); - assertEquals(1, results3.getTotalElements()); String deviceName3 = results3.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName3).isEqualTo(customerDevices.get(0).getName()); } @@ -1488,16 +1499,14 @@ public class EntityServiceTest extends AbstractControllerTest { // find by customer user with generic permissions apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); - PageData customerResult = entityService.findEntityDataByQuery(tenantId, customerId, query); + PageData customerResult = findByQueryAndCheck(customerId, query, 1); - assertEquals(1, customerResult.getTotalElements()); String customerResultName = customerResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(customerResultName).isEqualTo(TEST_CUSTOMER_NAME); // find by tenant user with customerId filter apiUsageStateFilter.setCustomerId(customerId); - PageData tenantResult = searchEntities(query); - assertEquals(1, tenantResult.getTotalElements()); + PageData tenantResult = findByQueryAndCheck(query, 1); String tenantResultName = tenantResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } @@ -1625,6 +1634,231 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } + @Test + public void testFindEntityDataByRelationQuery_blobEntity_customerLevel() { + final int deviceCnt = 2; + final int relationsCnt = 3; + final int blobEntitiesCnt = deviceCnt * relationsCnt; + + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("Customer Relation Query"); + customer = customerService.saveCustomer(customer); + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device relation query " + i); + device.setCustomerId(customer.getId()); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + } + + List blobEntities = new ArrayList<>(); + for (int i = 0; i < blobEntitiesCnt; i++) { + BlobEntity blobEntity = new BlobEntity(); + blobEntity.setName("Blob relation query " + i); + blobEntity.setTenantId(tenantId); + blobEntity.setContentType("image/png"); + blobEntity.setData(ByteBuffer.allocate(1024)); + blobEntity.setCustomerId(customer.getId()); + blobEntity.setType("Report"); + blobEntities.add(blobEntityService.saveBlobEntity(blobEntity)); + } + + for (int i = 0; i < deviceCnt; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(devices.get(i).getId()); + relationEntity.setTo(blobEntities.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("fileAttached"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions mergedUserPermissions = new MergedUserPermissions(Map.of(ALL, Set.of(Operation.ALL)), Collections.emptyMap()); + + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("fileAttached", Collections.singletonList(EntityType.BLOB_ENTITY)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + + for (Device device : devices) { + filter.setRootEntity(device.getId()); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + findByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedUserPermissions, query, relationsCnt); + /* + In order to be careful with updating Relation Query while adding new Entity Type, + this checkup will help to find place, where you could check the correctness of building query + */ + Assert.assertEquals(38, EntityType.values().length); + } + } + + @Test + public void testFindEntitiesByRelationEntityTypeFilterWithTenantGroupPermission() { + final int assetCount = 2; + final int relationsCnt = 4; + final int deviceEntitiesCnt = assetCount * relationsCnt; + + EntityGroup deviceGroup = new EntityGroup(); + deviceGroup.setName("Device Tenant Level Group"); + deviceGroup.setOwnerId(tenantId); + deviceGroup.setTenantId(tenantId); + deviceGroup.setType(EntityType.DEVICE); + deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); + + List assets = new ArrayList<>(); + for (int i = 0; i < assetCount; i++) { + Asset building = new Asset(); + building.setTenantId(tenantId); + building.setName("Building _" + i); + building.setType("building"); + building = assetService.saveAsset(building); + assets.add(building); + } + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceEntitiesCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Test device " + i); + device.setType("default"); + Device savedDevice = deviceService.saveDevice(device); + devices.add(savedDevice); + if (i % 2 == 0) { + entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); + } + } + + for (int i = 0; i < assetCount; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(assets.get(i).getId()); + relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("contains"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions groupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), + Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.READ)))); + + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); + + for (Asset asset : assets) { + filter.setRootEntity(asset.getId()); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); + countByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), groupOnlyPermission, query, relationsCnt / 2); + } + } + + @Test + public void testFindEntitiesWithRelationEntityTypeFilterByCustomerUser() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("Customer Relation Query"); + customer = customerService.saveCustomer(customer); + + final int assetCount = 2; + final int relationsCnt = 4; + final int deviceEntitiesCnt = assetCount * relationsCnt; + + EntityGroup deviceGroup = new EntityGroup(); + deviceGroup.setName("Device Tenant Level Group"); + deviceGroup.setOwnerId(customer.getId()); + deviceGroup.setTenantId(tenantId); + deviceGroup.setType(EntityType.DEVICE); + deviceGroup = entityGroupService.saveEntityGroup(tenantId, tenantId, deviceGroup); + + List assets = new ArrayList<>(); + for (int i = 0; i < assetCount; i++) { + Asset building = new Asset(); + building.setTenantId(tenantId); + building.setCustomerId(customer.getId()); + building.setName("Building _" + i); + building.setType("building"); + building = assetService.saveAsset(building); + assets.add(building); + } + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceEntitiesCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customer.getId()); + device.setName("Test device " + i); + device.setType("default"); + Device savedDevice = deviceService.saveDevice(device); + devices.add(savedDevice); + if (i % 2 == 0) { + entityGroupService.addEntityToEntityGroup(tenantId, deviceGroup.getId(), savedDevice.getId()); + } + } + + for (int i = 0; i < assetCount; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(assets.get(i).getId()); + relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("contains"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + MergedUserPermissions mergedGroupOnlyPermission = new MergedUserPermissions(Collections.emptyMap(), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + MergedUserPermissions mergedGenericOnlyPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Collections.emptyMap()); + MergedUserPermissions mergedGenericAndGroupPermission = new MergedUserPermissions(Map.of(Resource.ALL, Set.of(Operation.ALL)), Map.of(deviceGroup.getId(), new MergedGroupPermissionInfo(EntityType.DEVICE, Set.of(Operation.ALL)))); + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + + for (Asset asset : assets) { + filter.setRootEntity(asset.getId()); + + //check by user with generic permission + PageData relationsResult = findByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedGenericOnlyPermission, query, relationsCnt); + + //check by user with generic and group permission + PageData relationsResult1 = findByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); + countByQueryAndCheck(customer.getId(), mergedGenericAndGroupPermission, query, relationsCnt); + + //check by other customer user with group only permission + PageData relationsResult2 = findByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); + long relationsResultCnt2 = countByQueryAndCheck(otherCustomerId, mergedGroupOnlyPermission, query, relationsCnt / 2); + + Assert.assertEquals(relationsCnt / 2, relationsResult2.getData().size()); + Assert.assertEquals(relationsCnt / 2, relationsResultCnt2); + + //check by other customer user with generic and group only permission + PageData relationsResult3 = findByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); + long relationsResultCnt3 = countByQueryAndCheck(otherCustomerId, mergedGenericAndGroupPermission, query, relationsCnt / 2); + + Assert.assertEquals(relationsCnt / 2, relationsResult3.getData().size()); + Assert.assertEquals(relationsCnt / 2, relationsResultCnt3); + } + } + @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { @@ -2422,7 +2656,7 @@ public class EntityServiceTest extends AbstractControllerTest { return timeseriesService.save(tenantId, entityId, timeseries); } - private void createMultiRootHierarchy(List buildings, List apartments, + protected void createMultiRootHierarchy(List buildings, List apartments, Map> entityNameByTypeMap, Map childParentRelationMap) throws InterruptedException { for (int k = 0; k < 3; k++) { @@ -2507,7 +2741,7 @@ public class EntityServiceTest extends AbstractControllerTest { entityView.setEndTimeMs(256); entityView.setExternalId(new EntityViewId(UUID.randomUUID())); entityView.setAdditionalInfo(JacksonUtil.newObjectNode().put("test", "test")); - entityView = entityViewDao.save(tenantId, entityView); + entityView = entityViewService.saveEntityView(entityView); EntityViewTypeFilter entityViewTypeFilter = new EntityViewTypeFilter(); entityViewTypeFilter.setEntityViewNameFilter("test"); @@ -2518,21 +2752,18 @@ public class EntityServiceTest extends AbstractControllerTest { ); EntityDataQuery query = new EntityDataQuery(entityViewTypeFilter, pageLink, entityFields, Collections.emptyList(), null); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult.getData()).hasSize(1); + PageData relationsResult = findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 1); assertThat(relationsResult.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).isEqualTo(entityView.getName()); // find with non existing name entityViewTypeFilter.setEntityViewNameFilter("non-existing"); - PageData relationsResult2 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult2.getData()).hasSize(0); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); // find with non existing type entityViewTypeFilter.setEntityViewNameFilter(null); entityViewTypeFilter.setEntityViewTypes(Collections.singletonList("non-existing")); - PageData relationsResult3 = entityService.findEntityDataByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), query); - assertThat(relationsResult3.getData()).hasSize(0); + findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); } private PageData findByQuery(EntityDataQuery query) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 5a0db392ef..f1cf51bf8b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; @@ -275,6 +276,7 @@ public class FieldsUtil { return ApiUsageStateFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) + .customerId(entity.getEntityId().getEntityType() == EntityType.CUSTOMER ? entity.getEntityId().getId() : null) .entityId(entity.getEntityId()) .transportState(entity.getTransportState()) .dbStorageState(entity.getDbStorageState()) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 5383311e1f..24526205d8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -42,10 +42,11 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.repo.EdqRepository; +import org.thingsboard.server.edqs.state.EdqsStateService; import org.thingsboard.server.edqs.util.EdqsPartitionService; import org.thingsboard.server.edqs.util.VersionsStore; import org.thingsboard.server.gen.transport.TransportProtos; @@ -70,6 +71,7 @@ import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @EdqsComponent @@ -96,6 +98,8 @@ public class EdqsProcessor implements TbQueueHandler, private final VersionsStore versionsStore = new VersionsStore(); + private final AtomicInteger counter = new AtomicInteger(); // FIXME: TMP + @PostConstruct private void init() { consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); @@ -152,15 +156,17 @@ public class EdqsProcessor implements TbQueueHandler, responseTemplate.subscribe(withTopic(partitions, config.getRequestsTopic())); Set oldPartitions = event.getOldPartitions().get(new QueueKey(ServiceType.EDQS)); - Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() - .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); - if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { - log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + if (CollectionsUtil.isNotEmpty(oldPartitions)) { + Set removedPartitions = Sets.difference(oldPartitions, newPartitions).stream() + .map(tpi -> tpi.getPartition().orElse(-1)).collect(Collectors.toSet()); + if (config.getPartitioningStrategy() != EdqsPartitioningStrategy.TENANT && !removedPartitions.isEmpty()) { + log.warn("Partitions {} were removed but shouldn't be (due to NONE partitioning strategy)", removedPartitions); + } + repository.clearIf(tenantId -> { + Integer partition = partitionService.resolvePartition(tenantId); + return partition != null && removedPartitions.contains(partition); + }); } - repository.clearIf(tenantId -> { - Integer partition = partitionService.resolvePartition(tenantId); - return partition != null && removedPartitions.contains(partition); - }); } catch (Throwable t) { log.error("Failed to handle partition change event {}", event, t); } @@ -221,7 +227,11 @@ public class EdqsProcessor implements TbQueueHandler, } EdqsObject object = converter.deserialize(objectType, eventMsg.getData().toByteArray()); - log.info("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + log.debug("[{}] Processing event [{}] [{}] [{}] [{}]", tenantId, objectType, eventType, key, version); + int count = counter.incrementAndGet(); + if (count % 100000 == 0) { + log.info("Processed {} events", count); + } EdqsEvent event = EdqsEvent.builder() .tenantId(tenantId) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java index c2821161c7..7dbda46068 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/query/processor/ApiUsageStateQueryProcessor.java @@ -76,8 +76,7 @@ public class ApiUsageStateQueryProcessor extends AbstractSingleEntityTypeQueryPr @Override protected boolean matches(EntityData ed) { ApiUsageStateFields entityFields = (ApiUsageStateFields) ed.getFields(); - return super.matches(ed) && (filter.getCustomerId() != null ? entityFields.getEntityId().equals(filter.getCustomerId()) : - entityFields.getEntityId().equals(repository.getTenantId())); + return super.matches(ed) && (filter.getCustomerId() == null || filter.getCustomerId().equals(entityFields.getEntityId())); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index e8e8ee610e..04ef73faa1 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -71,7 +71,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private ScheduledExecutorService scheduler; private final VersionsStore versionsStore = new VersionsStore(); - private final AtomicInteger restoredCount = new AtomicInteger(); + private final AtomicInteger stateReadCount = new AtomicInteger(); + private final AtomicInteger eventsReadCount = new AtomicInteger(); @PostConstruct private void init() { @@ -79,7 +80,7 @@ public class KafkaEdqsStateService implements EdqsStateService { mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); - stateConsumer = MainQueueConsumerManager., QueueConfig>builder() + stateConsumer = MainQueueConsumerManager., QueueConfig>builder() // FIXME Slavik: if topic is empty .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.STATE.getTopic())) .config(QueueConfig.of(true, config.getPollInterval())) .msgPackProcessor((msgs, consumer, config) -> { @@ -88,8 +89,8 @@ public class KafkaEdqsStateService implements EdqsStateService { ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); edqsProcessor.process(msg, EdqsQueue.STATE); - if (restoredCount.incrementAndGet() % 1000 == 0) { - log.info("Processed {} msgs", restoredCount.get()); + if (stateReadCount.incrementAndGet() % 100000 == 0) { + log.info("[state] Processed {} msgs", stateReadCount.get()); } } catch (Throwable t) { log.error("Failed to process message: {}", queueMsg, t); @@ -103,7 +104,7 @@ public class KafkaEdqsStateService implements EdqsStateService { .scheduler(scheduler) .build(); - eventsConsumer = QueueConsumerManager.>builder() + eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer) -> { @@ -115,6 +116,9 @@ public class KafkaEdqsStateService implements EdqsStateService { if (msg.hasEventMsg()) { EdqsEventMsg eventMsg = msg.getEventMsg(); String key = eventMsg.getKey(); + if (eventsReadCount.incrementAndGet() % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", eventsReadCount.get()); + } if (eventMsg.hasVersion()) { if (!versionsStore.isNew(key, eventMsg.getVersion())) { return; @@ -153,14 +157,14 @@ public class KafkaEdqsStateService implements EdqsStateService { @Override public void restore(Set partitions) { - restoredCount.set(0); + stateReadCount.set(0); //TODO Slavik: do not support remote mode in monolith setup long startTs = System.currentTimeMillis(); log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait - log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), restoredCount.get()); + log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), stateReadCount.get()); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 165a54b300..ea9ba3f245 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -482,7 +482,7 @@ public class HashPartitionService implements PartitionService { private Set toTpiList(QueueKey queueKey, List partitions) { if (partitions == null) { - return null; + return Collections.emptySet(); } return partitions.stream() .map(partition -> buildTopicPartitionInfo(queueKey, partition)) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 167fbb1751..47c776c742 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -144,10 +144,11 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); + recordList = new ArrayList<>(256); records.forEach(record -> { recordList.add(record); if (stopWhenRead) { @@ -163,17 +164,18 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } } }); - if (endOffsets != null && endOffsets.isEmpty()) { - log.info("Reached end offsets for {}, stopping consumer", consumer.assignment()); - stop(); - } - return recordList; } + if (stopWhenRead && endOffsets.isEmpty()) { + log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + stop(); + } + return recordList; } private void onPartitionsAssigned() { if (stopWhenRead) { endOffsets = consumer.endOffsets(consumer.assignment()).entrySet().stream() + .filter(entry -> entry.getValue() > 0) .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); } } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index ca97792186..2fc7970fc9 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -38,7 +38,7 @@ public class DefaultStatsFactory implements StatsFactory { private static final Counter STUB_COUNTER = new StubCounter(); - @Autowired + @Autowired(required = false) // FIXME Slavik !!! private MeterRegistry meterRegistry; @Value("${metrics.enabled:false}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 2c37788173..6678be6009 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -22,14 +22,13 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.UUID; /** * @author Andrew Shvayka @@ -42,14 +41,14 @@ public interface AttributesDao { List findAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope); - PageData findAll(PageLink pageLink); - ListenableFuture save(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, AttributeKvEntry attribute); List> removeAll(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); List>> removeAllWithVersions(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, List keys); + List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize); + List findAllKeysByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); List findAllKeysByEntityIds(TenantId tenantId, List entityIds); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index d29b57c5c3..d668f51dbd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -18,8 +18,6 @@ package org.thingsboard.server.dao.relation; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -43,8 +41,6 @@ public interface RelationDao { List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); - PageData findAll(PageLink pageLink); - ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 90f49c57fd..de5651be00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -59,4 +59,12 @@ public interface AttributeKvRepository extends JpaRepository findAllKeysByEntityIdsAndAttributeType(@Param("entityIds") List entityIds, @Param("attributeType") int attributeType); + + @Query(value = "SELECT attribute_key, attribute_type, entity_id, bool_v, dbl_v, json_v, last_update_ts, long_v, str_v, version FROM attribute_kv WHERE (entity_id, attribute_type, attribute_key) > " + + "(:entityId, :attributeType, :attributeKey) ORDER BY entity_id, attribute_type, attribute_key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("attributeType") int attributeType, + @Param("attributeKey") int attributeKey, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index b344a0ca41..db166c44b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AttributeScope; @@ -31,8 +30,6 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -51,8 +48,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; @@ -157,9 +154,8 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public PageData findAll(PageLink pageLink) { - Page attributes = attributeKvRepository.findAll(DaoUtil.toPageable(pageLink)); - return DaoUtil.pageToPageData(attributes); + public List findNextBatch(UUID entityId, int attributeType, int attributeKey, int batchSize) { + return attributeKvRepository.findNextBatch(entityId, attributeType, attributeKey, batchSize); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 31443f7f47..c54b5561ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -693,6 +693,10 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO + (entityFilter.isMultiRoot() ? (", " + SELECT_RELATED_PARENT_ID) : "") + ", entity.entity_type as entity_type"; + /* + * FIXME: + * target entities are duplicated in result list, if search direction is TO and multiple relations are references to target entity + * */ String from = getQueryTemplate(entityFilter.getDirection(), entityFilter.isMultiRoot()); if (entityFilter.isMultiRoot()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 1df7115518..118c3c1bcc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -24,8 +24,6 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -129,10 +127,6 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } - @Override - public PageData findAll(PageLink pageLink) { - return DaoUtil.toPageData(relationRepository.findAll(DaoUtil.toPageable(pageLink))); - } @Override public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index d945f1f640..3c69ae095f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -85,6 +85,15 @@ public interface RelationRepository @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType and r.relationTypeGroup in :relationTypeGroups") void deleteByFromIdAndFromTypeAndRelationTypeGroupIn(@Param("fromId") UUID fromId, @Param("fromType") String fromType, @Param("relationTypeGroups") List relationTypeGroups); - @Query("SELECT e FROM RelationEntity e ORDER BY e.fromId, e.fromType, e.toId, e.toType, e.relationType, e.relationTypeGroup") - Page findAll(Pageable pageable); + @Query(value = "SELECT from_id, from_type, relation_type_group, relation_type, to_id, to_type, additional_info, version FROM relation" + + " WHERE (from_id, from_type, relation_type_group, relation_type, to_id, to_type) > " + + "(:fromId, :fromType, :relationTypeGroup, :relationType, :toId, :toType) ORDER BY " + + "from_id, from_type, relation_type_group, relation_type, to_id, to_type LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("fromId") UUID fromId, + @Param("fromType") String fromType, + @Param("relationTypeGroup") String relationTypeGroup, + @Param("relationType") String relationType, + @Param("toId") UUID toId, + @Param("toType") String toType, + @Param("batchSize") int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index 0e9cebd150..e6175409de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -171,9 +171,5 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return null; - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index fd1b180eef..dde89e107d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -188,10 +188,6 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return DaoUtil.pageToPageData(tsKvLatestRepository.findAll(DaoUtil.toPageable(pageLink, "entityId", "key"))); - } private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 833ffd185e..2c97ba30ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; @@ -46,4 +47,11 @@ public interface TsKvLatestRepository extends JpaRepository findAll(Pageable pageable); + + @Query(value = "SELECT entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v, version FROM ts_kv_latest WHERE (entity_id, key) > " + + "(:entityId, :key) ORDER BY entity_id, key LIMIT :batchSize", nativeQuery = true) + List findNextBatch(@Param("entityId") UUID entityId, + @Param("key") int key, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 99bf57a1db..6f00437672 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -103,10 +103,6 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } - @Override - public PageData findAllLatest(PageLink pageLink) { - return null; - } @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 1372f2f5d8..b2d7c0eff5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -54,5 +54,4 @@ public interface TimeseriesLatestDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - PageData findAllLatest(PageLink pageLink); } diff --git a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java index 1157f74de1..7fed88b0c3 100644 --- a/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java +++ b/edqs/src/main/java/org/thingsboard/server/edqs/ThingsboardEdqsApplication.java @@ -30,7 +30,7 @@ import java.util.Arrays; @EnableAsync @EnableScheduling @ComponentScan({"org.thingsboard.server.edqs", "org.thingsboard.server.queue.edqs", "org.thingsboard.server.queue.discovery", "org.thingsboard.server.queue.kafka", - "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment"}) + "org.thingsboard.server.queue.settings", "org.thingsboard.server.queue.environment", "org.thingsboard.server.common.stats"}) @Slf4j public class ThingsboardEdqsApplication { @@ -42,8 +42,8 @@ public class ThingsboardEdqsApplication { } // @Bean - public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { - return args -> { +// public ApplicationRunner runner(CSVLoader loader, EdqRepository edqRepository) { +// return args -> { // long startTs = System.currentTimeMillis(); // var loader = new TenantRepoLoader(new TenantRepo(TenantId.fromUUID(UUID.fromString("2a209df0-c7ff-11ea-a3e0-f321b0429d60")))); // loader.load(); @@ -103,8 +103,8 @@ public class ThingsboardEdqsApplication { // }); // Thread.sleep(5000); // } - }; - } +// }; +// } private static String[] updateArguments(String[] args) { if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index f5c83178c3..143a42fb68 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -52,15 +52,14 @@ queue: # For debug level print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" edqs: - enabled: "${TB_EDQS_ENABLED:true}" mode: "${TB_EDQS_MODE:local}" partitions: "${TB_EDQS_PARTITIONS:12}" + partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data requests_topic: "${TB_EDQS_REQUESTS_TOPIC:edqs.requests}" responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" - partitioning_strategy: "${TB_EDQS_PARTITIONING_STRATEGY:tenant}" # tenant or none. For 'none', each instance handles all partitions and duplicates all the data kafka: # Kafka Bootstrap nodes in "host:port" format bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" @@ -171,11 +170,11 @@ queue: # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for EDQS events topics. Partitions number must be the same as queue.edqs.partitions - edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" + edqs-events: "${TB_QUEUE_KAFKA_EDQS_EVENTS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS requests topic (default: 3 minutes retention). Partitions number must be the same as queue.edqs.partitions edqs-requests: "${TB_QUEUE_KAFKA_EDQS_REQUESTS_TOPIC_PROPERTIES:retention.ms:180000;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1}" # Kafka properties for EDQS state topic (infinite retention, compaction). Partitions number must be the same as queue.edqs.partitions - edqs-state: "${TB_QUEUE_KAFKA_EDQS_LATEST_EVENTS_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:1048576000;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" + edqs-state: "${TB_QUEUE_KAFKA_EDQS_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:-1;partitions:12;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" From fd42c51df1699c371350c264f5e7116df0ebda94 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 11:25:22 +0200 Subject: [PATCH 106/438] Calculated field add/edit basic implementation --- .../core/http/calculated-fields.service.ts | 56 +---- .../calculated-fields-table-config.ts | 74 ++++-- .../calculated-fields-table.component.ts | 23 +- ...lated-field-arguments-table.component.html | 106 +++++++++ ...lated-field-arguments-table.component.scss | 32 +++ ...culated-field-arguments-table.component.ts | 213 ++++++++++++++++++ .../calculated-field-dialog.component.html | 169 ++++++++++++++ .../calculated-field-dialog.component.ts | 122 ++++++++++ ...ulated-field-argument-panel.component.html | 198 ++++++++++++++++ ...ulated-field-argument-panel.component.scss | 29 +++ ...lculated-field-argument-panel.component.ts | 201 +++++++++++++++++ .../components/public-api.ts | 18 ++ .../home/components/home-components.module.ts | 19 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity/entity-autocomplete.component.html | 13 +- .../entity/entity-autocomplete.component.ts | 7 + .../entity-key-autocomplete.component.html | 39 ++++ .../entity-key-autocomplete.component.ts | 134 +++++++++++ .../shared/components/js-func.component.ts | 46 ++-- .../shared/models/calculated-field.models.ts | 94 +++++++- ui-ngx/src/app/shared/models/public-api.ts | 1 + .../src/app/shared/models/regex.constants.ts | 17 ++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 67 +++++- 24 files changed, 1572 insertions(+), 117 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/regex.constants.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 5df4a84949..dca0c9a3c7 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -16,7 +16,7 @@ import { Injectable } from '@angular/core'; import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable, of } from 'rxjs'; +import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; @@ -25,64 +25,26 @@ import { PageLink } from '@shared/models/page/page-link'; @Injectable({ providedIn: 'root' }) -// [TODO]: [Calculated fields] - implement when BE ready export class CalculatedFieldsService { - fieldsMock = [ - { - name: 'Calculated Field 1', - type: 'Simple', - configuration: { - expression: '1 + 2', - type: 'SIMPLE', - }, - entityId: '1', - id: { - id: '1', - } - }, - { - name: 'Calculated Field 2', - type: 'Script', - entityId: '2', - configuration: { - expression: '${power}', - type: 'SIMPLE', - }, - id: { - id: '2', - } - } - ] as any[]; - constructor( private http: HttpClient ) { } - public getCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(this.fieldsMock[0]); - // return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + public getCalculatedFieldById(calculatedFieldId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public saveCalculatedField(calculatedField: any, config?: RequestConfig): Observable { - return of(this.fieldsMock[1]); - // return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); + public saveCalculatedField(calculatedField: CalculatedField, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField', calculatedField, defaultHttpOptionsFromConfig(config)); } public deleteCalculatedField(calculatedFieldId: string, config?: RequestConfig): Observable { - return of(true); - // return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); + return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, - config?: RequestConfig): Observable> { - return of({ - data: this.fieldsMock, - totalPages: 1, - totalElements: 2, - hasNext: false, - }); - // return this.http.get>(`/api/calculatedField${pageLink.toQuery()}`, - // defaultHttpOptionsFromConfig(config)); + public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 22f742b9d8..2500c92336 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -23,41 +23,33 @@ import { TimePageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; -import { DialogService } from '@core/services/dialog.service'; import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { getCurrentAuthState } from '@core/auth/auth.selectors'; -import { ChangeDetectorRef, DestroyRef, ViewContainerRef } from '@angular/core'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; -import { EntityService } from '@core/http/entity.service'; +import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { DestroyRef } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; + readonly tenantId = getCurrentAuthUser(this.store).tenantId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, public entityId: EntityId = null, private store: Store, - private viewContainerRef: ViewContainerRef, - private overlay: Overlay, - private cd: ChangeDetectorRef, - private utilsService: UtilsService, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef, @@ -67,6 +59,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.addEntity = this.addCalculatedField.bind(this); + this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); + this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); + this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); + this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; @@ -97,8 +96,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - // // [TODO]: [Calculated fields] - implement edit - onAction: (_, entity) => {} + onAction: (_, entity) => this.editCalculatedField(entity) } ); } @@ -121,7 +119,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private editCalculatedField(calculatedField: CalculatedField): void { + this.getCalculatedFieldDialog(calculatedField, 'action.apply') + .afterClosed() + .pipe( + filter(Boolean), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + ) + .subscribe((res) => { + if (res) { + this.updateData(); + } + }); + } + + private getCalculatedFieldDialog(value = {}, buttonTitle = 'action.add') { + return this.dialog.open(CalculatedFieldDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + value, + buttonTitle, + entityId: this.entityId, + debugLimitsConfiguration: this.calculatedFieldsDebugPerTenantLimitsConfiguration, + tenantId: this.tenantId, + } + }) + } + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); @@ -149,7 +189,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...field, debugSettings })), catchError(() => of(null)), takeUntilDestroyed(this.destroyRef), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index f98d24bd52..853870982a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,24 +16,18 @@ import { ChangeDetectionStrategy, - ChangeDetectorRef, Component, DestroyRef, Input, OnInit, ViewChild, - ViewContainerRef, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; -import { EntityService } from '@core/http/entity.service'; -import { DialogService } from '@core/services/dialog.service'; import { TranslateService } from '@ngx-translate/core'; import { MatDialog } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { Overlay } from '@angular/cdk/overlay'; -import { UtilsService } from '@core/services/utils.service'; import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/calculated-fields-table-config'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { TbPopoverService } from '@shared/components/popover.service'; @@ -51,7 +45,6 @@ export class CalculatedFieldsTableComponent implements OnInit { set entityId(entityId: EntityId) { if (this.entityIdValue !== entityId) { this.entityIdValue = entityId; - this.entitiesTable.resetSortAndFilter(this.activeValue); if (!this.activeValue) { this.hasInitialized = true; } @@ -78,18 +71,12 @@ export class CalculatedFieldsTableComponent implements OnInit { private entityIdValue: EntityId; constructor(private calculatedFieldsService: CalculatedFieldsService, - private entityService: EntityService, - private dialogService: DialogService, private translate: TranslateService, private dialog: MatDialog, private store: Store, - private overlay: Overlay, - private viewContainerRef: ViewContainerRef, - private cd: ChangeDetectorRef, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, - private destroyRef: DestroyRef, - private utilsService: UtilsService) { + private destroyRef: DestroyRef) { } ngOnInit() { @@ -97,19 +84,13 @@ export class CalculatedFieldsTableComponent implements OnInit { this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( this.calculatedFieldsService, - this.entityService, - this.dialogService, this.translate, this.dialog, this.entityIdValue, this.store, - this.viewContainerRef, - this.overlay, - this.cd, - this.utilsService, this.durationLeft, this.popoverService, - this.destroyRef + this.destroyRef, ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html new file mode 100644 index 0000000000..99e68cf059 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -0,0 +1,106 @@ + +
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate }} + + + + } + + + + + {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + + + +
+ {{group.get('refEntityKey').get('key').value}} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
+ @if (errorText) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss new file mode 100644 index 0000000000..9507d9f012 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .inline-entity-autocomplete { + .mat-mdc-form-field-infix { + padding-top: 8px; + padding-bottom: 8px; + min-height: 40px; + width: auto; + .mdc-text-field__input, .mat-mdc-select { + font-weight: 400; + line-height: 20px; + } + } + a { + font-size: 14px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts new file mode 100644 index 0000000000..0ff6ecdf7f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -0,0 +1,213 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + DestroyRef, + effect, + forwardRef, + input, + Input, + Renderer2, + ViewContainerRef, +} from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { + ArgumentEntityType, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgument, + CalculatedFieldArgumentValue, + CalculatedFieldType, +} from '@shared/models/calculated-field.models'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; + +@Component({ + selector: 'tb-calculated-field-arguments-table', + templateUrl: './calculated-field-arguments-table.component.html', + styleUrls: [`calculated-field-arguments-table.component.scss`], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldArgumentsTableComponent), + multi: true + } + ], +}) +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { + + @Input() entityId: EntityId; + @Input() tenantId: string; + + calculatedFieldType = input() + + errorText = ''; + argumentsFormArray = this.fb.array([]); + keysPopupClosed = true; + + readonly entityTypeTranslations = entityTypeTranslations; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly EntityType = EntityType; + readonly ArgumentEntityType = ArgumentEntityType; + + private onChange: (argumentsObj: Record) => void = () => {}; + + constructor( + private fb: FormBuilder, + private popoverService: TbPopoverService, + private viewContainerRef: ViewContainerRef, + private destroyRef: DestroyRef, + private cd: ChangeDetectorRef, + private renderer: Renderer2 + ) { + this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { + this.onChange(this.getArgumentsObject()); + }); + effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + } + + registerOnChange(fn: (argumentsObj: Record) => void): void { + this.onChange = fn; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + return this.errorText ? { argumentsFormArray: false } : null; + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + + manageArgument($event: Event, matButton: MatButton, index?: number): void { + $event?.stopPropagation(); + const trigger = matButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const ctx = { + index, + argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + entityId: this.entityId, + calculatedFieldType: this.calculatedFieldType(), + buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', + tenantId: this.tenantId, + }; + this.keysPopupClosed = false; + const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + ctx, + {}, + {}, {}, true); + argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; + argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + argumentsPanelPopover.hide(); + const formGroup = this.getArgumentFormGroup(value); + if (isDefinedAndNotNull(index)) { + this.argumentsFormArray.setControl(index, formGroup); + } else { + this.argumentsFormArray.push(formGroup); + } + this.argumentsFormArray.markAsDirty(); + this.cd.markForCheck(); + }); + argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.keysPopupClosed = true; + }); + } + } + + getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + return this.fb.group({ + ...value, + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...(value.refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: value.refEntityId.entityType, disabled: true }], + id: [{ value: value.refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: value.refEntityKey.type, disabled: true }], + key: [{ value: value.refEntityKey.key, disabled: true }], + }), + }) + } + + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html new file mode 100644 index 0000000000..7d1373bebb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -0,0 +1,169 @@ + +
+ +

{{ 'entity.type-calculated-field' | translate}}

+ +
+ + +
+
+
+
{{ 'common.general' | translate }}
+
+ + {{ 'entity-field.title' | translate }} + + @if (fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched) { + + @if (fieldFormGroup.get('name').hasError('required')) { + {{ 'common.hint.name-required' | translate }} + } @else if (fieldFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.name-pattern' | translate }} + } @else if (fieldFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.name-max-length' | translate }} + } + + } + + +
+ + {{ 'common.type' | translate }} + + @for (type of fieldTypes; track type) { + {{ CalculatedFieldTypeTranslations.get(type) | translate}} + } + + +
+ +
+
{{ 'calculated-fields.arguments' | translate }}
+ +
+
+
{{ 'calculated-fields.expression' | translate }}*
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + @if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { + + @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { + {{ 'calculated-fields.hint.expression-required' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('pattern')) { + {{ 'calculated-fields.hint.expression-invalid' | translate }} + } @else if (configFormGroup.get('expressionSIMPLE').hasError('maxLength')) { + {{ 'calculated-fields.hint.expression-max-length' | translate }} + } + + } + + } @else { + + } +
+
+
{{ 'calculated-fields.output' | translate }}
+
+ + {{ 'calculated-fields.output-type' | translate }} + + @for (type of outputTypes; track type) { + {{ OutputTypeTranslations.get(type) | translate}} + } + + + @if (outputFormGroup.get('type').value === OutputType.Attribute) { + + {{ 'calculated-fields.output-type' | translate }} + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if (data.entityId.entityType === EntityType.DEVICE) { + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + + } +
+ @if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { + + + {{ (outputFormGroup.get('type').value === OutputType.Timeseries + ? 'calculated-fields.timeseries-key' + : 'calculated-fields.attribute-key') + | translate }} + + + @if (outputFormGroup.get('name').errors && outputFormGroup.get('name').touched) { + + @if (outputFormGroup.get('name').hasError('required')) { + {{ 'common.hint.key-required' | translate }} + } @else if (outputFormGroup.get('name').hasError('pattern')) { + {{ 'common.hint.key-pattern' | translate }} + } @else if (outputFormGroup.get('name').hasError('maxlength')) { + {{ 'common.hint.key-max-length' | translate }} + } + + } + + } +
+ +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts new file mode 100644 index 0000000000..99bf62ad46 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -0,0 +1,122 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { helpBaseUrl } from '@shared/models/constants'; +import { + CalculatedField, + CalculatedFieldConfiguration, + CalculatedFieldDialogData, + CalculatedFieldType, + CalculatedFieldTypeTranslations, + OutputType, + OutputTypeTranslations +} from '@shared/models/calculated-field.models'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; +import { EntityType } from '@shared/models/entity-type.models'; +import { map } from 'rxjs/operators'; +import { isObject } from '@core/utils'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; + +@Component({ + selector: 'tb-calculated-field-dialog', + templateUrl: './calculated-field-dialog.component.html', +}) +export class CalculatedFieldDialogComponent extends DialogComponent { + + fieldFormGroup = this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + type: [CalculatedFieldType.SIMPLE, [Validators.required]], + debugSettings: [], + configuration: this.fb.group({ + arguments: [{}], + expressionSIMPLE: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + expressionSCRIPT: [], + output: this.fb.group({ + name: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + type: [OutputType.Timeseries] + }), + }), + }); + + functionArgs$ = this.fieldFormGroup.get('configuration').valueChanges + .pipe( + map(configuration => isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + ); + + readonly OutputTypeTranslations = OutputTypeTranslations; + readonly OutputType = OutputType; + readonly AttributeScope = AttributeScope; + readonly EntityType = EntityType; + readonly CalculatedFieldType = CalculatedFieldType; + readonly ScriptLanguage = ScriptLanguage; + readonly helpLink = `${helpBaseUrl}/[TODO: ADD VALID LINK!!!]`; + readonly fieldTypes = Object.values(CalculatedFieldType) as CalculatedFieldType[]; + readonly outputTypes = Object.values(OutputType) as OutputType[]; + readonly CalculatedFieldTypeTranslations = CalculatedFieldTypeTranslations; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, + public dialogRef: MatDialogRef, + public fb: UntypedFormBuilder) { + super(store, router, dialogRef); + this.applyDialogData(); + this.outputFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleScopeByOutputType(type)); + this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + } + + get configFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration') as FormGroup; + } + + get outputFormGroup(): FormGroup { + return this.fieldFormGroup.get('configuration').get('output') as FormGroup; + } + + cancel(): void { + this.dialogRef.close(null); + } + + add(): void { + if (this.fieldFormGroup.valid) { + const { configuration, type, ...rest } = this.fieldFormGroup.value; + const { expressionSIMPLE, expressionSCRIPT, ...restConfig } = configuration; + this.dialogRef.close({ configuration: { ...restConfig, type, expression: configuration['expression'+type] }, ...rest, type }); + } + } + + private applyDialogData(): void { + const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value; + const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; + this.fieldFormGroup.patchValue({ configuration: { ...restConfig, ['expression'+type]: expression }, ...value }); + } + + private toggleScopeByOutputType(type: OutputType): void { + this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html new file mode 100644 index 0000000000..26a347aaa2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -0,0 +1,198 @@ + +
+
+
{{ 'calculated-fields.argument-settings' | translate }}
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
+ + + @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { + + warning + + } + +
+
+ +
+
{{ 'entity.entity-type' | translate }}
+ + + @for (type of argumentEntityTypes; track type) { + {{ ArgumentEntityTypeTranslations.get(type) | translate }} + } + + +
+ @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { +
+
{{ 'calculated-fields.device-name' | translate }}
+ +
+ } @else if (entityType === ArgumentEntityType.Customer) { +
+
{{ 'calculated-fields.customer-name' | translate }}
+ +
+ } + + +
+
{{ 'calculated-fields.argument-type' | translate }}
+ + + @for (type of argumentTypes; track type) { + {{ ArgumentTypeTranslations.get(type) | translate }} + } + + @if (refEntityKeyFormGroup.get('type').hasError('required') && refEntityKeyFormGroup.get('type').touched) { + + warning + + } + +
+ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} + + @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } + + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { +
+
{{ 'calculated-fields.default-value' | translate }}
+
+ + + +
+
+ } @else { +
+
{{ 'calculated-fields.time-window' | translate }}
+
+ + + {{ 'common.suffix.ms' | translate }} + +
+
+
+
{{ 'calculated-fields.limit' | translate }}
+
+ + + +
+
+ } +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..a784909b92 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .mat-mdc-form-field { + width: 100%; + } +} + +:host ::ng-deep { + .entity-autocomplete { + .mat-mdc-form-field { + width: 100%; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts new file mode 100644 index 0000000000..2ba38beee5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -0,0 +1,201 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { + ArgumentEntityType, + ArgumentEntityTypeTranslations, + ArgumentType, + ArgumentTypeTranslations, + CalculatedFieldArgumentValue, + CalculatedFieldType +} from '@shared/models/calculated-field.models'; +import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { EntityType } from '@shared/models/entity-type.models'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { DatasourceType } from '@shared/models/widget.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { EntityFilter } from '@shared/models/query/query.models'; +import { AliasFilterType } from '@shared/models/alias.models'; +import { merge } from 'rxjs'; + +@Component({ + selector: 'tb-calculated-field-argument-panel', + templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] +}) +export class CalculatedFieldArgumentPanelComponent extends PageComponent implements OnInit { + + @Input() popover: TbPopoverComponent; + @Input() buttonTitle: string; + @Input() index: number; + @Input() argument: CalculatedFieldArgumentValue; + @Input() entityId: EntityId; + @Input() tenantId: string; + @Input() calculatedFieldType: CalculatedFieldType; + + @ViewChild('timeseriesInput') timeseriesInput: ElementRef; + + argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); + + argumentFormGroup = this.fb.group({ + argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + refEntityId: this.fb.group({ + entityType: [ArgumentEntityType.Current], + id: [''] + }), + refEntityKey: this.fb.group({ + type: [ArgumentType.LatestTelemetry, [Validators.required]], + key: [''], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + }), + defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], + limit: [null], + timeWindow: [null], + }); + + argumentTypes: ArgumentType[]; + entityFilter: EntityFilter; + keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges + .pipe( + startWith(this.refEntityIdFormGroup.get('entityType').value), + map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) + ); + + readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; + readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; + readonly ArgumentType = ArgumentType; + readonly DataKeyType = DataKeyType; + readonly EntityType = EntityType; + readonly datasourceType = DatasourceType; + readonly ArgumentTypeTranslations = ArgumentTypeTranslations; + readonly AttributeScope = AttributeScope; + + constructor( + private fb: FormBuilder, + ) { + super(); + + this.observeEntityFilterChanges(); + this.observeEntityTypeChanges() + this.observeEntityKeyChanges(); + } + + get entityType(): ArgumentEntityType { + return this.argumentFormGroup.get('refEntityId').get('entityType').value; + } + + get refEntityIdFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityId') as FormGroup; + } + + get refEntityKeyFormGroup(): FormGroup { + return this.argumentFormGroup.get('refEntityKey') as FormGroup; + } + + ngOnInit(): void { + this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.updateEntityFilter(this.argument.refEntityId?.entityType, true); + this.toggleByEntityKeyType(this.argument.refEntityKey?.type); + this.setInitialEntityKeyType(); + + this.argumentTypes = Object.values(ArgumentType) + .filter(type => type !== ArgumentType.Rolling || this.calculatedFieldType === CalculatedFieldType.SCRIPT); + } + + saveArgument(): void { + this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + } + + cancel(): void { + this.popover.hide(); + } + + private toggleByEntityKeyType(type: ArgumentType): void { + const isAttribute = type === ArgumentType.Attribute; + const isRolling = type === ArgumentType.Rolling; + this.argumentFormGroup.get('refEntityKey').get('scope')[isAttribute? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('limit')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('timeWindow')[isRolling? 'enable' : 'disable']({ emitEvent: false }); + this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); + } + + private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + let entityId: EntityId; + switch (entityType) { + case ArgumentEntityType.Current: + entityId = this.entityId + break; + case ArgumentEntityType.Tenant: + entityId = { + id: this.tenantId, + entityType: EntityType.TENANT + }; + break; + default: + entityId = this.argumentFormGroup.get('refEntityId').value as any; + } + if (onInit) { + this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); + } + this.entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } + + private observeEntityFilterChanges(): void { + merge( + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges, + this.argumentFormGroup.get('refEntityId').get('id').valueChanges.pipe(filter(Boolean)), + this.argumentFormGroup.get('refEntityKey').get('scope').valueChanges, + ) + .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .subscribe(() => this.updateEntityFilter(this.entityType)); + } + + private observeEntityTypeChanges(): void { + this.argumentFormGroup.get('refEntityId').get('entityType').valueChanges + .pipe(distinctUntilChanged(), takeUntilDestroyed()) + .subscribe(type => { + this.argumentFormGroup.get('refEntityId').get('id').setValue(''); + this.argumentFormGroup.get('refEntityId') + .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + }); + } + + private observeEntityKeyChanges(): void { + this.argumentFormGroup.get('refEntityKey').get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleByEntityKeyType(type)); + } + + private setInitialEntityKeyType(): void { + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argument.refEntityKey?.type === ArgumentType.Rolling) { + const typeControl = this.argumentFormGroup.get('refEntityKey').get('type'); + typeControl.setValue(null); + typeControl.markAsTouched(); + typeControl.updateValueAndValidity(); + } + } + + protected readonly ArgumentEntityType = ArgumentEntityType; +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts new file mode 100644 index 0000000000..c3d1ede02e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -0,0 +1,18 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export * from './dialog/calculated-field-dialog.component'; +export * from './arguments-table/calculated-field-arguments-table.component'; diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 1a6b9c08e0..f60ea15407 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -185,6 +185,16 @@ import { EntityChipsComponent } from '@home/components/entity/entity-chips.compo import { DashboardViewComponent } from '@home/components/dashboard-view/dashboard-view.component'; import { CalculatedFieldsTableComponent } from '@home/components/calculated-fields/calculated-fields-table.component'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; +import { CalculatedFieldDialogComponent } from '@home/components/calculated-fields/components/dialog/calculated-field-dialog.component'; +import { + EntityDebugSettingsButtonComponent +} from '@home/components/entity/debug/entity-debug-settings-button.component'; +import { + CalculatedFieldArgumentsTableComponent +} from '@home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component'; +import { + CalculatedFieldArgumentPanelComponent +} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; @NgModule({ declarations: @@ -330,6 +340,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], imports: [ CommonModule, @@ -341,7 +354,8 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; SnmpDeviceProfileTransportModule, StatesControllerModule, DeviceCredentialsModule, - DeviceProfileCommonModule + DeviceProfileCommonModule, + EntityDebugSettingsButtonComponent ], exports: [ RouterTabsComponent, @@ -468,6 +482,9 @@ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; EntityChipsComponent, DashboardViewComponent, CalculatedFieldsTableComponent, + CalculatedFieldDialogComponent, + CalculatedFieldArgumentsTableComponent, + CalculatedFieldArgumentPanelComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts index 618650ac32..fbbf124629 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.ts @@ -19,6 +19,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { DeviceInfo } from '@shared/models/device.models'; import { EntityTabsComponent } from '../../components/entity/entity-tabs.component'; +import { EntityType } from '@shared/models/entity-type.models'; @Component({ selector: 'tb-device-tabs', @@ -27,6 +28,8 @@ import { EntityTabsComponent } from '../../components/entity/entity-tabs.compone }) export class DeviceTabsComponent extends EntityTabsComponent { + readonly EntityType = EntityType; + constructor(protected store: Store) { super(store); } @@ -34,5 +37,4 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } - } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index c194e04290..44a218b455 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -17,10 +17,11 @@ --> - {{ label | translate }} + {{ label | translate }} {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} + + warning + + } + + @for (key of filteredKeys$ | async; track key) { + + } + + diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts new file mode 100644 index 0000000000..2a33a74786 --- /dev/null +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -0,0 +1,134 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, effect, ElementRef, forwardRef, input, ViewChild, } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { map, startWith, switchMap } from 'rxjs/operators'; +import { combineLatest, of, Subject } from 'rxjs'; +import { EntityService } from '@core/http/entity.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { EntitiesKeysByQuery } from '@shared/models/entity.models'; +import { EntityFilter } from '@shared/models/query/query.models'; + +@Component({ + selector: 'tb-entity-key-autocomplete', + templateUrl: './entity-key-autocomplete.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityKeyAutocompleteComponent), + multi: true + } + ], + host: { + class: 'w-full' + } +}) +export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { + + @ViewChild('keyInput') keyInput: ElementRef; + + entityFilter = input.required(); + dataKeyType = input.required(); + keyScopeType = input(); + + keyControl = this.fb.control('', [Validators.required]); + searchText = ''; + keyInputSubject = new Subject(); + + private onChange: (value: string) => void; + private cachedResult: EntitiesKeysByQuery; + + keys$ = this.keyInputSubject.asObservable() + .pipe( + switchMap(() => { + return this.cachedResult ? of(this.cachedResult) : this.entityService.findEntityKeysByQuery({ + pageLink: { page: 0, pageSize: 100 }, + entityFilter: this.entityFilter(), + }, this.dataKeyType() === DataKeyType.attribute, this.dataKeyType() === DataKeyType.timeseries, this.keyScopeType()); + }), + map(result => { + this.cachedResult = result; + switch (this.dataKeyType()) { + case DataKeyType.attribute: + return result.attribute; + case DataKeyType.timeseries: + return result.timeseries; + default: + return []; + } + }), + ); + + filteredKeys$ = combineLatest([this.keys$, this.keyControl.valueChanges.pipe(startWith(''))]) + .pipe( + map(([keys, searchText = '']) => { + this.searchText = searchText; + return searchText ? keys.filter(item => item.toLowerCase().includes(searchText.toLowerCase())) : keys; + }) + ); + + constructor( + private fb: FormBuilder, + private entityService: EntityService, + ) { + this.keyControl.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(value => this.onChange(value)); + effect(() => { + if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { + this.cachedResult = null; + this.searchText = ''; + } + }); + } + + clear(): void { + this.keyControl.patchValue('', {emitEvent: true}); + setTimeout(() => { + this.keyInput.nativeElement.blur(); + this.keyInput.nativeElement.focus(); + }, 0); + } + + registerOnChange(onChange: (value: string) => void): void { + this.onChange = onChange; + } + + registerOnTouched(_): void {} + + validate(): ValidationErrors | null { + return this.keyControl.valid ? null : { keyControl: false }; + } + + writeValue(value: string): void { + this.keyControl.patchValue(value, {emitEvent: false}); + } +} diff --git a/ui-ngx/src/app/shared/components/js-func.component.ts b/ui-ngx/src/app/shared/components/js-func.component.ts index 8e500bdbe9..57c15a5484 100644 --- a/ui-ngx/src/app/shared/components/js-func.component.ts +++ b/ui-ngx/src/app/shared/components/js-func.component.ts @@ -20,9 +20,11 @@ import { ElementRef, forwardRef, Input, + OnChanges, OnDestroy, OnInit, Renderer2, + SimpleChanges, ViewChild, ViewContainerRef, ViewEncapsulation @@ -67,7 +69,7 @@ import { catchError } from 'rxjs/operators'; ], encapsulation: ViewEncapsulation.None }) -export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, Validator { +export class JsFuncComponent implements OnInit, OnChanges, OnDestroy, ControlValueAccessor, Validator { @ViewChild('javascriptEditor', {static: true}) javascriptEditorElmRef: ElementRef; @@ -177,6 +179,13 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, private http: HttpClient) { } + ngOnChanges(changes: SimpleChanges): void { + if (changes.functionArgs) { + this.updateFunctionArgsString(); + this.updateFunctionLabel(); + } + } + ngOnInit(): void { if (this.functionTitle || this.label) { this.hideBrackets = true; @@ -184,22 +193,6 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, if (!this.resultType || this.resultType.length === 0) { this.resultType = 'nocheck'; } - if (this.functionArgs) { - this.functionArgs.forEach((functionArg) => { - if (this.functionArgsString.length > 0) { - this.functionArgsString += ', '; - } - this.functionArgsString += functionArg; - }); - } - if (this.functionTitle) { - this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; - } else if (this.label) { - this.functionLabel = this.label; - } else { - this.functionLabel = - `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; - } const editorElement = this.javascriptEditorElmRef.nativeElement; let editorOptions: Partial = { mode: 'ace/mode/javascript', @@ -329,6 +322,25 @@ export class JsFuncComponent implements OnInit, OnDestroy, ControlValueAccessor, ); } + private updateFunctionArgsString(): void { + this.functionArgsString = ''; + if (this.functionArgs) { + this.functionArgsString = this.functionArgs.join(', '); + } + } + + private updateFunctionLabel(): void { + if (this.functionTitle) { + this.functionLabel = `${this.functionTitle}: f(${this.functionArgsString})`; + } else if (this.label) { + this.functionLabel = this.label; + } else { + this.functionLabel = + `function ${this.functionName ? this.functionName : ''}(${this.functionArgsString})${this.hideBrackets ? '' : ' {'}`; + } + this.cd.markForCheck(); + } + validateOnSubmit(): Observable { if (!this.disabled) { this.cleanupJsErrors(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 253bc58f39..73b3ecd861 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -13,29 +13,109 @@ /// See the License for the specific language governing permissions and /// limitations under the License. /// + import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; import { BaseData } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; +import { EntityId } from '@shared/models/id/entity-id'; +import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { - type: CalculatedFieldType; debugSettings?: EntityDebugSettings; externalId?: string; configuration: CalculatedFieldConfiguration; + type: CalculatedFieldType; } export enum CalculatedFieldType { SIMPLE = 'SIMPLE', - COMPLEX = 'COMPLEX', + SCRIPT = 'SCRIPT', } +export const CalculatedFieldTypeTranslations = new Map( + [ + [CalculatedFieldType.SIMPLE, 'calculated-fields.type.simple'], + [CalculatedFieldType.SCRIPT, 'calculated-fields.type.script'], + ] +) + export interface CalculatedFieldConfiguration { - type: CalculatedFieldConfigType; + type: CalculatedFieldType; expression: string; - arguments: Record; + arguments: Record; } -export enum CalculatedFieldConfigType { - SIMPLE = 'SIMPLE', - SCRIPT = 'SCRIPT', +export enum ArgumentEntityType { + Current = 'CURRENT', + Device = 'DEVICE', + Asset = 'ASSET', + Customer = 'CUSTOMER', + Tenant = 'TENANT', +} + +export const ArgumentEntityTypeTranslations = new Map( + [ + [ArgumentEntityType.Current, 'calculated-fields.argument-current'], + [ArgumentEntityType.Device, 'calculated-fields.argument-device'], + [ArgumentEntityType.Asset, 'calculated-fields.argument-asset'], + [ArgumentEntityType.Customer, 'calculated-fields.argument-customer'], + [ArgumentEntityType.Tenant, 'calculated-fields.argument-tenant'], + ] +) + +export enum ArgumentType { + Attribute = 'ATTRIBUTE', + LatestTelemetry = 'TS_LATEST', + Rolling = 'TS_ROLLING', +} + +export enum OutputType { + Attribute = 'ATTRIBUTES', + Timeseries = 'TIME_SERIES', +} + +export const OutputTypeTranslations = new Map( + [ + [OutputType.Attribute, 'calculated-fields.attribute'], + [OutputType.Timeseries, 'calculated-fields.timeseries'], + ] +) + +export const ArgumentTypeTranslations = new Map( + [ + [ArgumentType.Attribute, 'calculated-fields.attribute'], + [ArgumentType.LatestTelemetry, 'calculated-fields.latest-telemetry'], + [ArgumentType.Rolling, 'calculated-fields.rolling'], + ] +) + +export interface CalculatedFieldArgument { + refEntityKey: RefEntityKey; + defaultValue?: string; + refEntityId?: RefEntityKey; + limit?: number; + timeWindow?: number; +} + +export interface RefEntityKey { + key: string; + type: ArgumentType; + scope?: AttributeScope; +} + +export interface RefEntityKey { + entityType: ArgumentEntityType; + id: string; +} + +export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { + argumentName: string; +} + +export interface CalculatedFieldDialogData { + value: CalculatedField; + buttonTitle: string; + entityId: EntityId; + debugLimitsConfiguration: string; + tenantId: string; } diff --git a/ui-ngx/src/app/shared/models/public-api.ts b/ui-ngx/src/app/shared/models/public-api.ts index 736d885810..c48a3286cd 100644 --- a/ui-ngx/src/app/shared/models/public-api.ts +++ b/ui-ngx/src/app/shared/models/public-api.ts @@ -61,3 +61,4 @@ export * from './widgets-bundle.model'; export * from './window-message.model'; export * from './usage.models'; export * from './query/query.models'; +export * from './regex.constants'; diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts new file mode 100644 index 0000000000..55742cefd4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -0,0 +1,17 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 5b40dadf5f..5825998434 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -224,6 +224,7 @@ import { IntervalOptionsConfigPanelComponent } from '@shared/components/time/int import { GroupingIntervalOptionsComponent } from '@shared/components/time/aggregation/grouping-interval-options.component'; import { JsFuncModulesComponent } from '@shared/components/js-func-modules.component'; import { JsFuncModuleRowComponent } from '@shared/components/js-func-module-row.component'; +import { EntityKeyAutocompleteComponent } from '@shared/components/entity/entity-key-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -432,7 +433,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) ImageGalleryDialogComponent, WidgetButtonComponent, HexInputComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ], imports: [ CommonModule, @@ -694,7 +696,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) EmbedImageDialogComponent, ImageGalleryDialogComponent, WidgetButtonComponent, - ScadaSymbolInputComponent + ScadaSymbolInputComponent, + EntityKeyAutocompleteComponent, ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index beb6734451..be31b4f70b 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -996,6 +996,7 @@ "failures": "Failures", "entity": "entity", "rule-node": "rule node", + "calculated-field": "calculated field", "hint": { "main": "All node debug messages rate limited with:", "main-limited": "All {{entity}} debug messages will be rate-limited, with a maximum of {{msg}} messages allowed per {{time}}.", @@ -1007,7 +1008,56 @@ "expression": "Expression", "no-found": "No calculated fields found", "list": "{ count, plural, =1 {One calculated field} other {List of # calculated fields} }", - "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected" + "selected-fields": "{ count, plural, =1 {1 calculated field} other {# calculated fields} } selected", + "type": { + "simple": "Simple", + "script": "Script" + }, + "arguments": "Arguments", + "argument-name": "Argument name", + "datasource": "Datasource", + "add-argument": "Add argument", + "no-arguments": "No arguments configured", + "argument-settings": "Argument settings", + "argument-current": "Current entity", + "argument-current-tenant": "Current tenant", + "argument-device": "Device", + "argument-asset": "Asset", + "argument-customer": "Customer", + "argument-tenant": "Current tenant", + "argument-type": "Argument type", + "attribute": "Attribute", + "timeseries-key": "Time series key", + "device-name": "Device name", + "latest-telemetry": "Latest telemetry", + "rolling": "Rolling", + "attribute-scope": "Attribute scope", + "server-attributes": "Server attributes", + "client-attributes": "Client attributes", + "shared-attributes": "Shared attributes", + "attribute-key": "Attribute key", + "default-value": "Default value", + "limit": "Limit", + "time-window": "Time window", + "customer-name": "Customer name", + "timeseries": "Time series", + "output": "Output", + "output-type": "Output type", + "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", + "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", + "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", + "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", + "hint": { + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-empty": "Arguments should not be empty.", + "expression-required": "Expression is required.", + "expression-invalid": "Expression is invalid", + "expression-max-length": "Expression length should be less than 255 characters.", + "argument-name-required": "Argument name is required.", + "argument-name-pattern": "Argument name is invalid.", + "argument-name-max-length": "Argument name should be less than 256 characters.", + "argument-type-required": "Argument type is required." + } }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", @@ -1035,6 +1085,7 @@ "common": { "name": "Name", "type": "Type", + "general": "General", "username": "Username", "password": "Password", "enter-username": "Enter username", @@ -1047,7 +1098,19 @@ "open-details-page": "Open details page", "not-found": "Not found", "documentation": "Documentation", - "time-left": "{{time}} left" + "time-left": "{{time}} left", + "suffix": { + "s": "s", + "ms": "ms" + }, + "hint": { + "name-required": "Name is required.", + "name-pattern": "Name is invalid.", + "name-max-length": "Name should be less than 256 characters.", + "key-required": "Key is required.", + "key-pattern": "Key is invalid.", + "key-max-length": "Key should be less than 256 characters." + } }, "content-type": { "json": "Json", From c1c9fb4f5aa9ee4fa5c369ba51436b194d908b1f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 11:29:38 +0200 Subject: [PATCH 107/438] implemented calculated field debug events persistence --- .../server/actors/ActorSystemContext.java | 52 +++++++++++-------- ...CalculatedFieldEntityMessageProcessor.java | 42 ++++++++++----- ...alculatedFieldManagerMessageProcessor.java | 20 +++---- ...efaultCalculatedFieldExecutionService.java | 15 ++++-- .../ctx/state/BaseCalculatedFieldState.java | 19 ++++--- .../cf/ctx/state/CalculatedFieldCtx.java | 42 +++++++++++++-- .../cf/ctx/state/CalculatedFieldState.java | 4 ++ .../ctx/state/ScriptCalculatedFieldState.java | 6 +++ .../ctx/state/SimpleCalculatedFieldState.java | 15 ++---- .../cf/DefaultTbCalculatedFieldService.java | 10 ++++ common/proto/src/main/proto/queue.proto | 3 ++ ...efaultNativeCalculatedFieldRepository.java | 3 ++ .../engine/api/AttributesSaveRequest.java | 18 ++++++- .../engine/api/TimeseriesSaveRequest.java | 18 ++++++- .../engine/telemetry/TbMsgAttributesNode.java | 2 + .../engine/telemetry/TbMsgTimeseriesNode.java | 2 + 16 files changed, 194 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 09488bfe4e..daa6744fd9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.TbMsg; @@ -105,6 +106,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; @@ -129,6 +131,7 @@ import org.thingsboard.server.service.transport.TbCoreToTransportService; import java.io.PrintWriter; import java.io.StringWriter; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; @@ -744,29 +747,34 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, TbMsg tbMsg, Throwable error) { - if (checkLimits(tenantId, tbMsg, error)) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder event = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(tbMsg.getOriginator()) - .msgId(tbMsg.getId()) - .msgType(tbMsg.getType()) - .arguments(JacksonUtil.toString(arguments)) - .result(tbMsg.getData()); - - if (error != null) { - event.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(event.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString(arguments)); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..6426be8a3c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -18,19 +18,18 @@ package org.thingsboard.server.actors.calculatedField; import com.google.common.util.concurrent.ListenableFuture; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -53,9 +52,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** @@ -111,9 +108,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM callback.onSuccess(CALLBACKS_PER_CF); } else { if (proto.getTsDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else if (proto.getAttrDataCount() > 0) { - processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIds, callback, mapToArguments(ctx, msg.getEntityId(), proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -136,27 +133,30 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processTelemetry(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getTsDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processAttributes(CalculatedFieldCtx ctx, CalculatedFieldTelemetryMsgProto proto, List cfIdList, MultipleTbCallback callback) { - processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList())); + processArgumentValuesUpdate(ctx, cfIdList, callback, mapToArguments(ctx, proto.getScope(), proto.getAttrDataList()), toTbMsgId(proto), toTbMsgType(proto)); } @SneakyThrows private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, - Map newArgValues) { + Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + } } else { callback.onSuccess(); // State was updated but no calculation performed; } @@ -183,13 +183,27 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +235,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..603ab96a8e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,14 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; +import org.thingsboard.common.util.DebugModeUtil; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -38,16 +36,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -101,6 +90,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 9c8aafc47d..8f948aa265 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -833,7 +833,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); List entries = request.getEntries(); List versions = result.getVersions(); for (int i = 0; i < entries.size(); i++) { @@ -849,7 +849,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds()); + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); List entries = request.getEntries(); for (int i = 0; i < entries.size(); i++) { @@ -862,7 +862,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return msg.build(); } - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds) { + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); @@ -878,6 +878,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + return telemetryMsg; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index d623043cee..f5f681b505 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,19 +15,18 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.NoArgsConstructor; + import java.util.HashMap; import java.util.List; import java.util.Map; +@NoArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; protected Map arguments; - public BaseCalculatedFieldState() { - this.arguments = new HashMap<>(); - } - public BaseCalculatedFieldState(List requiredArguments) { this.requiredArguments = requiredArguments; this.arguments = new HashMap<>(); @@ -35,7 +34,12 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { @Override public Map getArguments() { - return this.arguments; + return arguments; + } + + @Override + public List getRequiredArguments() { + return requiredArguments; } @Override @@ -53,7 +57,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { if (existingEntry == null) { validateNewEntry(newEntry); - arguments.put(key, newEntry.copy()); + arguments.put(key, newEntry); stateUpdated = true; } else { stateUpdated = existingEntry.updateEntry(newEntry); @@ -70,7 +74,6 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { !arguments.containsValue(TsRollingArgumentEntry.EMPTY); } - protected void validateNewEntry(ArgumentEntry newEntry) { - } + protected abstract void validateNewEntry(ArgumentEntry newEntry); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 42536188ce..33421a7d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -17,6 +17,8 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.Data; import net.objecthunter.exp4j.Expression; +import net.objecthunter.exp4j.ExpressionBuilder; +import org.mvel2.MVEL; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; @@ -33,7 +35,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -45,6 +46,8 @@ import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { + private CalculatedField calculatedField; + private CalculatedFieldId cfId; private TenantId tenantId; private EntityId entityId; @@ -61,7 +64,11 @@ public class CalculatedFieldCtx { private CalculatedFieldScriptEngine calculatedFieldScriptEngine; private ThreadLocal customExpression; + private boolean initialized; + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + this.calculatedField = calculatedField; + this.cfId = calculatedField.getId(); this.tenantId = calculatedField.getTenantId(); this.entityId = calculatedField.getEntityId(); @@ -88,10 +95,28 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; - if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) { - this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + } + + public void init() { + if (CalculatedFieldType.SCRIPT.equals(cfType)) { + try { + this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); + initialized = true; + } catch (Exception e) { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax.", e); + } } else { - this.customExpression = new ThreadLocal<>(); + if (isValidExpression(expression)) { + this.customExpression = ThreadLocal.withInitial(() -> + new ExpressionBuilder(expression) + .implicitMultiplication(true) + .variables(this.arguments.keySet()) + .build() + ); + initialized = true; + } else { + throw new RuntimeException("Failed to init calculated field ctx. Invalid expression syntax."); + } } } @@ -108,6 +133,15 @@ public class CalculatedFieldCtx { ); } + private boolean isValidExpression(String expression) { + try { + MVEL.compileExpression(expression); + return true; + } catch (Exception e) { + return false; + } + } + public boolean matches(List values, AttributeScope scope) { return matchesAttributes(mainEntityArguments, values, scope); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java index 4b7918cc03..173d299da6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.List; import java.util.Map; @JsonTypeInfo( @@ -40,9 +41,12 @@ public interface CalculatedFieldState { Map getArguments(); + List getRequiredArguments(); + boolean updateState(Map argumentValues); ListenableFuture performCalculation(CalculatedFieldCtx ctx); + @JsonIgnore boolean isReady(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 290fd95370..a3c7efb388 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -31,6 +32,7 @@ import java.util.TreeMap; @Data @Slf4j +@NoArgsConstructor public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { public ScriptCalculatedFieldState(List requiredArguments) { @@ -42,6 +44,10 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { return CalculatedFieldType.SCRIPT; } + @Override + protected void validateNewEntry(ArgumentEntry newEntry) { + } + @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { arguments.forEach((key, argumentEntry) -> { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 59b3009bb4..8b8fe6e8c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -18,8 +18,7 @@ package org.thingsboard.server.service.cf.ctx.state; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Data; -import net.objecthunter.exp4j.Expression; -import net.objecthunter.exp4j.ExpressionBuilder; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -28,6 +27,7 @@ import java.util.List; import java.util.Map; @Data +@NoArgsConstructor public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { public SimpleCalculatedFieldState(List requiredArguments) { @@ -48,16 +48,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { @Override public ListenableFuture performCalculation(CalculatedFieldCtx ctx) { - String expression = ctx.getExpression(); - ThreadLocal customExpression = ctx.getCustomExpression(); - var expr = customExpression.get(); - if (expr == null) { - expr = new ExpressionBuilder(expression) - .implicitMultiplication(true) - .variables(this.arguments.keySet()) - .build(); - customExpression.set(expr); - } + var expr = ctx.getCustomExpression().get(); for (Map.Entry entry : this.arguments.entrySet()) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index c8c589691b..184092720c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -58,6 +58,10 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = calculatedField.getTenantId(); try { + if (ActionType.UPDATED.equals(actionType)) { + CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); + checkForEntityChange(existingCf, calculatedField); + } checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); checkArgumentSize(calculatedField.getConfiguration()); @@ -98,6 +102,12 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } } + private void checkForEntityChange(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { + if (!oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId())) { + throw new IllegalArgumentException("Changing the calculated field target entity after initialization is prohibited."); + } + } + private void checkEntityExistence(TenantId tenantId, EntityId entityId) { switch (entityId.getEntityType()) { case ASSET, DEVICE, ASSET_PROFILE, DEVICE_PROFILE -> diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..53045fc147 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -799,6 +799,9 @@ message CalculatedFieldTelemetryMsgProto { repeated TsKvProto tsData = 9; AttributeScopeProto scope = 10; repeated AttributeValueProto attrData = 11; + int64 tbMsgIdMSB = 12; + int64 tbMsgIdLSB = 13; + string tbMsgType = 14; } message CalculatedFieldLinkedTelemetryMsgProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index fcbf4d4dd0..daca6d5056 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -78,6 +79,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF int configurationVersion = (int) row.get("configuration_version"); JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; + String debugSettings = (String) row.get("debug_settings"); Object externalIdObj = row.get("external_id"); CalculatedField calculatedField = new CalculatedField(); @@ -90,6 +92,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); + calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); return calculatedField; diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java index 406b2d0d5c..74f3468f49 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java @@ -28,8 +28,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @ToString @@ -42,6 +44,8 @@ public class AttributesSaveRequest { private final List entries; private final boolean notifyDevice; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -56,6 +60,8 @@ public class AttributesSaveRequest { private List entries; private boolean notifyDevice = true; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; private FutureCallback callback; Builder() {} @@ -108,6 +114,16 @@ public class AttributesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -128,7 +144,7 @@ public class AttributesSaveRequest { } public AttributesSaveRequest build() { - return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, callback); + return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java index 95eb788e5f..957b0cf108 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java @@ -27,8 +27,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; import java.util.List; +import java.util.UUID; @Getter @AllArgsConstructor(access = AccessLevel.PRIVATE) @@ -42,6 +44,8 @@ public class TimeseriesSaveRequest { private final boolean saveLatest; private final boolean onlyLatest; private final List previousCalculatedFieldIds; + private final UUID tbMsgId; + private final TbMsgType tbMsgType; private final FutureCallback callback; public static Builder builder() { @@ -59,6 +63,8 @@ public class TimeseriesSaveRequest { private boolean saveLatest = true; private boolean onlyLatest; private List previousCalculatedFieldIds; + private UUID tbMsgId; + private TbMsgType tbMsgType; Builder() {} @@ -111,6 +117,16 @@ public class TimeseriesSaveRequest { return this; } + public Builder tbMsgId(UUID tbMsgId) { + this.tbMsgId = tbMsgId; + return this; + } + + public Builder tbMsgType(TbMsgType tbMsgType) { + this.tbMsgType = tbMsgType; + return this; + } + public Builder callback(FutureCallback callback) { this.callback = callback; return this; @@ -131,7 +147,7 @@ public class TimeseriesSaveRequest { } public TimeseriesSaveRequest build() { - return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, callback); + return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, previousCalculatedFieldIds, tbMsgId, tbMsgType, callback); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java index ae2fce6575..925d70daa1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java @@ -126,6 +126,8 @@ public class TbMsgAttributesNode implements TbNode { .entries(attributes) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(callback) .build()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java index 89f7844313..3287cc825f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java @@ -113,6 +113,8 @@ public class TbMsgTimeseriesNode implements TbNode { .ttl(ttl) .saveLatest(!config.isSkipLatestPersistence()) .previousCalculatedFieldIds(msg.getPreviousCalculatedFieldIds()) + .tbMsgId(msg.getId()) + .tbMsgType(msg.getInternalType()) .callback(new TelemetryNodeCallback(ctx, msg)) .build()); } From c01024f6da8b66b2a78adcde6884d40bfd15cabc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 12:11:09 +0200 Subject: [PATCH 108/438] changed endpoint --- .../server/controller/CalculatedFieldController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 124dd8d710..96a97aeeff 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -94,11 +94,11 @@ public class CalculatedFieldController extends BaseController { return calculatedField; } - @ApiOperation(value = "Get Calculated Fields (getCalculatedFieldsByEntityId)", + @ApiOperation(value = "Get Calculated Fields by Entity Id (getCalculatedFieldsByEntityId)", notes = "Fetch the Calculated Fields based on the provided Entity Id." ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/{entityType}/{entityId}/calculatedField", params = {"pageSize", "page"}, method = RequestMethod.GET) + @RequestMapping(value = "/{entityType}/{entityId}/calculatedFields", params = {"pageSize", "page"}, method = RequestMethod.GET) @ResponseBody public PageData getCalculatedFieldsByEntityId( @Parameter(description = ENTITY_TYPE_PARAM_DESCRIPTION, required = true, schema = @Schema(defaultValue = "DEVICE")) @PathVariable("entityType") String entityType, From a82d3690f3e282bb3ef18aec90bda15f69a387ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 12:30:36 +0200 Subject: [PATCH 109/438] Changed getAll endpoint and refactoring --- .../core/http/calculated-fields.service.ts | 5 +- .../calculated-fields-table-config.ts | 16 +- ...lated-field-arguments-table.component.html | 138 +++++++++--------- ...lated-field-arguments-table.component.scss | 1 + ...culated-field-arguments-table.component.ts | 101 +++++++------ .../calculated-field-dialog.component.html | 4 +- .../calculated-field-dialog.component.ts | 4 +- ...ulated-field-argument-panel.component.html | 53 +++---- ...lculated-field-argument-panel.component.ts | 9 +- .../components/public-api.ts | 1 + .../pages/device/device-tabs.component.html | 3 +- .../pages/device/device-tabs.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 9 +- .../shared/models/calculated-field.models.ts | 1 + 14 files changed, 179 insertions(+), 170 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index dca0c9a3c7..10ad366e5b 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -21,6 +21,7 @@ import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; +import { EntityId } from '@shared/models/id/entity-id'; @Injectable({ providedIn: 'root' @@ -43,8 +44,8 @@ export class CalculatedFieldsService { return this.http.delete(`/api/calculatedField/${calculatedFieldId}`, defaultHttpOptionsFromConfig(config)); } - public getCalculatedFields(pageLink: PageLink, config?: RequestConfig): Observable> { - return this.http.get>(`/api/calculatedFields${pageLink.toQuery()}`, + public getCalculatedFields({ entityType, id }: EntityId, pageLink: PageLink, config?: RequestConfig): Observable> { + return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 2500c92336..ae025a13f6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -40,6 +40,7 @@ import { CalculatedFieldDialogComponent } from './components/public-api'; export class CalculatedFieldsTableConfig extends EntityTableConfig { + // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration'] || '1:1'; readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; @@ -66,9 +67,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: TimePageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); - this.deleteEntityTitle = (field) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); + this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); this.deleteEntitiesTitle = count => this.translate.instant('calculated-fields.delete-multiple-title', {count}); this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); @@ -102,7 +103,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { - return this.calculatedFieldsService.getCalculatedFields(pageLink); + return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { @@ -134,10 +135,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField} as any)), + switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) .subscribe((res) => { if (res) { @@ -148,10 +148,9 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField} as any)), + switchMap((updatedCalculatedField) => this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), ) .subscribe((res) => { if (res) { @@ -160,7 +159,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -172,6 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig{{ 'common.type' | translate }}
{{ 'entity.key' | translate }}
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id').value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate }} - - - - } - +
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + @if (group.get('refEntityId')?.get('id').value) { + - - - {{ ArgumentTypeTranslations.get(group.get('refEntityKey').get('type').value) | translate }} + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - -
- {{group.get('refEntityKey').get('key').value}} -
- - + -
- - -
+ } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } + + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ +
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} + } +
@if (errorText) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 9507d9f012..8695ee4068 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -25,6 +25,7 @@ line-height: 20px; } } + a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 0ff6ecdf7f..1dedc5e80e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -43,9 +43,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType, } from '@shared/models/calculated-field.models'; -import { - CalculatedFieldArgumentPanelComponent -} from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/public-api'; import { MatButton } from '@angular/material/button'; import { TbPopoverService } from '@shared/components/popover.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -87,7 +85,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; - private onChange: (argumentsObj: Record) => void = () => {}; + private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, @@ -98,59 +96,27 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private renderer: Renderer2 ) { this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { - this.onChange(this.getArgumentsObject()); + this.propagateChange(this.getArgumentsObject()); }); effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); } registerOnChange(fn: (argumentsObj: Record) => void): void { - this.onChange = fn; + this.propagateChange = fn; } registerOnTouched(_): void {} validate(): ValidationErrors | null { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE - && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { - this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; - } else if (!this.argumentsFormArray.controls.length) { - this.errorText = 'calculated-fields.hint.arguments-empty'; - } else { - this.errorText = ''; - } + this.updateErrorText(); return this.errorText ? { argumentsFormArray: false } : null; } - private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); - const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; - acc[argumentName] = argument; - return acc; - }, {} as Record); - } - - writeValue(argumentsObj: Record): void { - this.argumentsFormArray.clear(); - Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], - ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); - }); + onDelete(index: number): void { + this.argumentsFormArray.removeAt(index); + this.argumentsFormArray.markAsDirty(); } - manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); const trigger = matButton._elementRef.nativeElement; @@ -189,7 +155,51 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } } - getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private updateErrorText(): void { + if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { + this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; + } else if (!this.argumentsFormArray.controls.length) { + this.errorText = 'calculated-fields.hint.arguments-empty'; + } else { + this.errorText = ''; + } + } + + private getArgumentsObject(): Record { + return this.argumentsFormArray.controls.reduce((acc, control) => { + const rawValue = control.getRawValue(); + const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; + acc[argumentName] = argument; + return acc; + }, {} as Record); + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + this.populateArgumentsFormArray(argumentsObj) + } + + private populateArgumentsFormArray(argumentsObj: Record): void { + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + ...argumentsObj[key], + ...(argumentsObj[key].refEntityId ? { + refEntityId: this.fb.group({ + entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], + id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], + }), + } : {}), + refEntityKey: this.fb.group({ + type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], + key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], + }), + }) as AbstractControl); + }); + } + + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], @@ -205,9 +215,4 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), }) } - - onDelete(index: number): void { - this.argumentsFormArray.removeAt(index); - this.argumentsFormArray.markAsDirty(); - } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 7d1373bebb..e6adc1b4d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -68,7 +68,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" - [calculatedFieldType]="fieldFormGroup.get('type').valueChanges | async" + [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -96,7 +96,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: ADD VALID LINK HERE!!!" + helpId="[TODO]: [Calculated Fields] add valid link" /> }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 99bf62ad46..7f9af5cead 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -47,7 +47,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent - @if (argumentFormGroup.get('argumentName').hasError('required') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern') && argumentFormGroup.get('argumentName').touched) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength') && argumentFormGroup.get('argumentName').touched) { - - warning - + @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } @@ -118,7 +120,7 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
@@ -143,6 +145,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
this.updateEntityFilter(this.entityType)); @@ -196,6 +197,4 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme typeControl.updateValueAndValidity(); } } - - protected readonly ArgumentEntityType = ArgumentEntityType; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index c3d1ede02e..bc89e4dc6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -16,3 +16,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; +export * from './panel/calculated-field-argument-panel.component'; diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index ab59f412cd..6a4f87ca6b 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -32,8 +32,7 @@ [entityName]="entity.name"> - + { - readonly EntityType = EntityType; - constructor(protected store: Store) { super(store); } @@ -37,4 +34,5 @@ export class DeviceTabsComponent extends EntityTabsComponent { ngOnInit() { super.ngOnInit(); } + } diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts index 2a33a74786..fc13238db0 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.ts @@ -47,9 +47,6 @@ import { EntityFilter } from '@shared/models/query/query.models'; multi: true } ], - host: { - class: 'w-full' - } }) export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Validator { @@ -63,7 +60,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val searchText = ''; keyInputSubject = new Subject(); - private onChange: (value: string) => void; + private propagateChange: (value: string) => void; private cachedResult: EntitiesKeysByQuery; keys$ = this.keyInputSubject.asObservable() @@ -101,7 +98,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val ) { this.keyControl.valueChanges .pipe(takeUntilDestroyed()) - .subscribe(value => this.onChange(value)); + .subscribe(value => this.propagateChange(value)); effect(() => { if (this.keyScopeType() || this.entityFilter() && this.dataKeyType()) { this.cachedResult = null; @@ -119,7 +116,7 @@ export class EntityKeyAutocompleteComponent implements ControlValueAccessor, Val } registerOnChange(onChange: (value: string) => void): void { - this.onChange = onChange; + this.propagateChange = onChange; } registerOnTouched(_): void {} diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 73b3ecd861..54798a23e7 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -25,6 +25,7 @@ export interface CalculatedField extends Omit, 'labe externalId?: string; configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; + entityId: EntityId; } export enum CalculatedFieldType { From d3216f3ee78ce154ea3a4c8a224003728497c84f Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 13:11:12 +0200 Subject: [PATCH 110/438] Changed table load --- .../calculated-fields-table.component.ts | 62 ++++++------------- 1 file changed, 20 insertions(+), 42 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 853870982a..4c22ad2896 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -18,8 +18,8 @@ import { ChangeDetectionStrategy, Component, DestroyRef, - Input, - OnInit, + effect, + input, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -39,36 +39,14 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; styleUrls: ['./calculated-fields-table.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class CalculatedFieldsTableComponent implements OnInit { - - @Input() - set entityId(entityId: EntityId) { - if (this.entityIdValue !== entityId) { - this.entityIdValue = entityId; - if (!this.activeValue) { - this.hasInitialized = true; - } - } - } - - @Input() - set active(active: boolean) { - if (this.activeValue !== active) { - this.activeValue = active; - if (this.activeValue && this.hasInitialized) { - this.hasInitialized = false; - this.entitiesTable.updateData(); - } - } - } +export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - calculatedFieldsTableConfig: CalculatedFieldsTableConfig; + active = input(); + entityId = input(); - private activeValue = false; - private hasInitialized = false; - private entityIdValue: EntityId; + calculatedFieldsTableConfig: CalculatedFieldsTableConfig; constructor(private calculatedFieldsService: CalculatedFieldsService, private translate: TranslateService, @@ -77,20 +55,20 @@ export class CalculatedFieldsTableComponent implements OnInit { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private destroyRef: DestroyRef) { - } - ngOnInit() { - this.hasInitialized = !this.activeValue; - - this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( - this.calculatedFieldsService, - this.translate, - this.dialog, - this.entityIdValue, - this.store, - this.durationLeft, - this.popoverService, - this.destroyRef, - ); + effect(() => { + if (this.active()) { + this.calculatedFieldsTableConfig = new CalculatedFieldsTableConfig( + this.calculatedFieldsService, + this.translate, + this.dialog, + this.entityId(), + this.store, + this.durationLeft, + this.popoverService, + this.destroyRef, + ); + } + }); } } From 83b338c697ff6ab7188886e273df4736dd1c3d80 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 31 Jan 2025 13:54:53 +0200 Subject: [PATCH 111/438] Entity lifecycle implementation --- .../server/actors/app/AppActor.java | 5 +- .../CalculatedFieldEntityActor.java | 6 + .../CalculatedFieldEntityDeleteMsg.java | 44 +++ ...CalculatedFieldEntityMessageProcessor.java | 51 ++- .../CalculatedFieldManagerActor.java | 8 +- ...alculatedFieldManagerMessageProcessor.java | 215 +++++++++++- .../EntityInitCalculatedFieldMsg.java | 41 +++ .../server/actors/tenant/TenantActor.java | 3 +- .../cf/CalculatedFieldExecutionService.java | 21 +- ...efaultCalculatedFieldExecutionService.java | 314 +++--------------- .../cf/ctx/CalculatedFieldStateService.java | 2 +- .../cf/ctx/state/CalculatedFieldCtx.java | 9 +- .../cf/ctx/state/RocksDBStateService.java | 4 +- .../entitiy/EntityStateSourcingListener.java | 10 +- ...faultTbCalculatedFieldConsumerService.java | 100 +----- .../queue/DefaultTbClusterService.java | 150 ++++----- .../server/cluster/TbClusterService.java | 4 +- .../server/queue/TbQueueCallback.java | 14 + .../server/common/msg/MsgType.java | 14 +- .../cf/CalculatedFieldEntityLifecycleMsg.java | 37 +++ .../msg/plugin/ComponentLifecycleMsg.java | 20 ++ .../server/common/util/ProtoUtils.java | 41 ++- common/proto/src/main/proto/queue.proto | 35 +- 23 files changed, 595 insertions(+), 553 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java create mode 100644 application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 9124dc0a9a..8a6907c107 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -115,12 +115,13 @@ public class AppActor extends ContextAwareActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: + //TODO: use priority from the message body. For example, messages about CF lifecycle are important and Device lifecycle are not. + // same for the Linked telemetry. onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index a5461c6152..cebe6b6a60 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -53,6 +53,12 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; + case CF_ENTITY_INIT_CF_MSG: + processor.process((EntityInitCalculatedFieldMsg) msg); + break; + case CF_ENTITY_DELETE_MSG: + processor.process((CalculatedFieldEntityDeleteMsg) msg); + break; case CF_ENTITY_TELEMETRY_MSG: processor.process((EntityCalculatedFieldTelemetryMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java new file mode 100644 index 0000000000..8df6a63a82 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityDeleteMsg.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityDeleteMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId entityId; + private final TbCallback callback; + + public CalculatedFieldEntityDeleteMsg(TenantId tenantId, + EntityId entityId, + TbCallback callback) { + this.tenantId = tenantId; + this.entityId = entityId; + this.callback = callback; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_DELETE_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index cea3cce791..b8f357fd87 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -88,6 +89,29 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM states.put(msg.getId().cfId(), msg.getState()); } + public void process(EntityInitCalculatedFieldMsg msg) { + var cfCtx = msg.getCtx(); + if (msg.isForceReinit()) { + states.remove(cfCtx.getCfId()); + } + var cfState = getOrInitState(cfCtx); + processStateIfReady(cfCtx, Collections.singletonList(cfCtx.getCfId()), cfState, msg.getCallback()); + } + + public void process(CalculatedFieldEntityDeleteMsg msg) { + if (this.entityId.equals(msg.getEntityId())) { + MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); + states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + ctx.stop(ctx.getSelf()); + } else { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var state = states.remove(cfId); + if (state != null) { + cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + } + } + } + public void process(EntityCalculatedFieldTelemetryMsg msg) { var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); @@ -152,15 +176,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } CalculatedFieldState state = getOrInitState(ctx); if (state.updateState(newArgValues)) { - if (state.isReady()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, callback); } else { callback.onSuccess(CALLBACKS_PER_CF); } @@ -183,13 +201,24 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } + @SneakyThrows + private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, TbCallback callback) { + if (state.isReady()) { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); // State was updated but no calculation performed; + } + cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + } + private Map mapToArguments(CalculatedFieldCtx ctx, List data) { return mapToArguments(ctx.getMainEntityArguments(), data); } private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, List data) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, data); @@ -221,7 +250,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private Map mapToArguments(CalculatedFieldCtx ctx, EntityId entityId, AttributeScopeProto scope, List attrDataList) { var argNames = ctx.getLinkedEntityArguments().get(entityId); - if(argNames.isEmpty()) { + if (argNames.isEmpty()) { return Collections.emptyMap(); } return mapToArguments(argNames, scope, attrDataList); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 1c198c660d..22909dd5af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -22,6 +22,7 @@ import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; @@ -66,8 +67,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_STATE_RESTORE_MSG: processor.onStateRestoreMsg((CalculatedFieldStateRestoreMsg) msg); break; - case CF_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); + case CF_ENTITY_LIFECYCLE_MSG: + processor.onEntityLifecycleMsg((CalculatedFieldEntityLifecycleMsg) msg); break; case CF_TELEMETRY_MSG: processor.onTelemetryMsg((CalculatedFieldTelemetryMsg) msg); @@ -75,9 +76,6 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { case CF_LINKED_TELEMETRY_MSG: processor.onLinkedTelemetryMsg((CalculatedFieldLinkedTelemetryMsg) msg); break; - case CF_ENTITY_UPDATE_MSG: -// processor.onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg); - break; default: return false; } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 4bce7cb322..e82f407e3a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -16,16 +16,13 @@ package org.thingsboard.server.actors.calculatedField; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; @@ -36,18 +33,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; -import org.thingsboard.server.queue.discovery.HashPartitionService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -78,7 +70,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfService; + private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; @@ -88,7 +81,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.tenantId = tenantId; @@ -104,7 +98,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cfCtx); msg.getCallback().onSuccess(); } @@ -112,7 +106,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link); + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link); msg.getCallback().onSuccess(); } @@ -124,6 +118,168 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } } + public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + var entityType = msg.getData().getEntityId().getEntityType(); + var event = msg.getData().getEvent(); + switch (entityType) { + case CALCULATED_FIELD: { + switch (event) { + case CREATED: + onCfCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onCfUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onCfDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + case DEVICE: + case ASSET: { + switch (event) { + case CREATED: + onEntityCreated(msg.getData(), msg.getCallback()); + break; + case UPDATED: + onEntityUpdated(msg.getData(), msg.getCallback()); + break; + case DELETED: + onEntityDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } + default: { + msg.getCallback().onSuccess(); + } + } + } + + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { + EntityId entityId = msg.getEntityId(); + var entityIdFields = getCalculatedFieldsByEntityId(entityId); + var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + var fieldsCount = entityIdFields.size() + profileIdFields.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + entityIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + profileIdFields.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + + private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); + var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); + var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); + if (fieldsCount > 0) { + MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); + var entityId = msg.getEntityId(); + oldProfileCfs.forEach(ctx -> deleteCfForEntity(entityId, ctx.getCfId(), callback)); + newProfileCfs.forEach(ctx -> initCfForEntity(entityId, ctx, true, multiCallback)); + } else { + callback.onSuccess(); + } + } + } + + private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); + } + + private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + if (calculatedFields.containsKey(cfId)) { + log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cf = cfDaoService.findById(msg.getTenantId(), cfId); + if (cf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + calculatedFields.put(cf.getId(), cfCtx); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + initCf(cfCtx, callback, false); + } + } + } + + private void onCfUpdated(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var oldCfCtx = calculatedFields.get(cfId); + if (oldCfCtx == null) { + onCfCreated(msg, callback); + } else { + var newCf = cfDaoService.findById(msg.getTenantId(), cfId); + if (newCf == null) { + log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + calculatedFields.put(newCf.getId(), newCfCtx); + List oldCfList = entityIdCalculatedFields.get(newCf.getId()); + List newCfList = new ArrayList<>(oldCfList.size()); + boolean found = false; + for (CalculatedFieldCtx oldCtx : oldCfList) { + if (oldCtx.getCfId().equals(newCf.getId())) { + newCfList.add(newCfCtx); + found = true; + } else { + newCfList.add(oldCtx); + } + } + if (!found) { + newCfList.add(newCfCtx); + } + entityIdCalculatedFields.put(newCf.getId(), newCfList); + // We use copy on write lists to safely pass the reference to another actor for the iteration. + // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) + if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + initCf(newCfCtx, callback, true); + } + } + } + + } + + private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + var cfId = new CalculatedFieldId(msg.getEntityId().getId()); + var cfCtx = calculatedFields.remove(cfId); + if (cfCtx == null) { + log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + callback.onSuccess(); + } else { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + //TODO: no need to do this if we cache all created actors and know which one belong to us; + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> deleteCfForEntity(entityId, cfId, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + deleteCfForEntity(entityId, cfId, callback); + } + } + } + public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); // 2 = 1 for CF processing + 1 for links processing @@ -140,7 +296,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware List linkedCalculatedFields = filterCalculatedFieldLinks(msg); var linksSize = linkedCalculatedFields.size(); if (linksSize > 0) { - cfService.pushMsgToLinks(msg, linkedCalculatedFields, callback); + cfExecService.pushMsgToLinks(msg, linkedCalculatedFields, callback); } else { callback.onSuccess(); } @@ -237,6 +393,33 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return entities; } + private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { + EntityId entityId = cfCtx.getEntityId(); + EntityType entityType = cfCtx.getEntityId().getEntityType(); + if (isProfileEntity(entityType)) { + var entityIds = getEntitiesByProfile(entityId); + if (!entityIds.isEmpty()) { + var multiCallback = new MultipleTbCallback(entityIds.size(), callback); + entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + } else { + callback.onSuccess(); + } + } else { + initCfForEntity(entityId, cfCtx, forceStateReinit, callback); + } + } + + private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); + } + + private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); + } + + private static boolean isProfileEntity(EntityType entityType) { + return EntityType.DEVICE_PROFILE.equals(entityType) || EntityType.ASSET_PROFILE.equals(entityType); + } private EntityId getProfileId(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java new file mode 100644 index 0000000000..00a378c90a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/EntityInitCalculatedFieldMsg.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.actors.calculatedField; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; + +import java.util.List; + +@Data +public class EntityInitCalculatedFieldMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final CalculatedFieldCtx ctx; + private final TbCallback callback; + private final boolean forceReinit; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_INIT_CF_MSG; + } +} diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 6ae10aaece..1b008f462a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,12 +170,11 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: - case CF_UPDATE_MSG: + case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; case CF_TELEMETRY_MSG: case CF_LINKED_TELEMETRY_MSG: - case CF_ENTITY_UPDATE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, false); break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index d8daae2e64..19f60165cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -20,16 +20,11 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -50,25 +45,11 @@ public interface CalculatedFieldExecutionService { void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); -// void pushEntityUpdateMsg(TransportProtos.CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - /* ===================================================== */ - - void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback); - - void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback); - - void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback); - - + void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 19561b73b9..9aca0d5b3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,7 +39,6 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; @@ -65,7 +64,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; @@ -75,11 +73,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; -import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; @@ -122,7 +117,6 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -273,6 +267,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas stateService.persistState(stateId, state, callback); } + @Override + public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { + stateService.removeState(calculatedFieldEntityCtxId, callback); + } + @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); @@ -330,271 +329,44 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } - @Override - public void pushCalculatedFieldLifecycleMsgToQueue(CalculatedField calculatedField, ComponentLifecycleMsgProto proto) { - EntityId entityId = calculatedField.getEntityId(); - ToCalculatedFieldMsg msg = ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(proto).build(); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - Set tpiSet = calculatedFieldCache.getEntitiesByProfile(calculatedField.getTenantId(), entityId).stream() - .map(targetEntityId -> partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, targetEntityId)) - .collect(Collectors.toSet()); - tpiSet.forEach(tpi -> clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, null)); - } - default -> throw new IllegalArgumentException("Entity type '" + calculatedField.getId().getEntityType() - + "' does not support calculated fields."); - } - } - - @Override - public void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); - ComponentLifecycleEvent event = proto.getEvent(); - if (ComponentLifecycleEvent.DELETED.equals(event)) { - log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - calculatedFieldCache.evict(calculatedFieldId); - onCalculatedFieldDelete(calculatedFieldId, callback); - callback.onSuccess(); - } - CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (ComponentLifecycleEvent.UPDATED.equals(event)) { - log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); - boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); - if (!shouldReinit) { - return; - } - } - if (cf != null) { - calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); - EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - switch (entityId.getEntityType()) { - case ASSET, DEVICE -> { - log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntity(calculatedFieldCtx, entityId, callback); - } - case ASSET_PROFILE, DEVICE_PROFILE -> { - log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); - Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { - calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { - initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); - }); - }); - } - default -> - throw new IllegalArgumentException("Entity type '" + calculatedFieldId.getEntityType() + "' does not support calculated fields."); - } - } else { - //Calculated field was probably deleted while message was in queue; - log.warn("Calculated field not found, possibly deleted: {}", calculatedFieldId); - callback.onSuccess(); - } - callback.onSuccess(); - log.info("Successfully processed calculated field message for calculatedFieldId: [{}]", calculatedFieldId); - } catch (Exception e) { - log.trace("Failed to process calculated field msg: [{}]", proto, e); - callback.onFailure(e); - } - } - - private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); - boolean shouldReinit = true; - if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); - } else { - calculatedFieldCache.updateCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); - callback.onSuccess(); - shouldReinit = false; - } - return shouldReinit; - } - - private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { - try { - cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> { - if (ctxId.cfId().equals(calculatedFieldId)) { - stateService.removeState(ctxId); - return true; - } - return false; - }); - } catch (Exception e) { - log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); - callback.onFailure(e); - } - } - - private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) { - if (oldCalculatedField == null) { - return true; - } - boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId()); - boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType()); - boolean argumentsChanged = !oldCalculatedField.getConfiguration().getArguments().equals(newCalculatedField.getConfiguration().getArguments()); - - return entityIdChanged || typeChanged || argumentsChanged; - } - - @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); - EntityId entityId = request.getEntityId(); - - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = request.getTenantId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - - if (tpi.isMyPartition()) { - - processCalculatedFields(request, entityId); - processCalculatedFields(request, getProfileId(tenantId, entityId)); - - Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(request, tpiStatesToUpdate); - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsgProto = buildLinkedTelemetryMsgProto(proto, ctxIds); - clusterService.pushMsgToCalculatedFields(topicPartitionInfo, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setLinkedTelemetryMsg(linkedTelemetryMsgProto).build(), null); - }); - } - } else { - clusterService.pushMsgToCalculatedFields(tpi, UUID.randomUUID(), ToCalculatedFieldMsg.newBuilder().setTelemetryMsg(proto).build(), null); - } - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); - } - } - - private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } - } - - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - - calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) - .forEach(link -> { - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - EntityId targetEntityId = ctx.getEntityId(); - - if (isProfileEntity(targetEntityId)) { - calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); - }); - } else { - processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); - } - }); - } - - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); - if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getTenantId(), ctx.getCfId(), targetEntity)); - } - } - - @Override - public void onTelemetryUpdate(CalculatedFieldLinkedTelemetryMsgProto proto, TbCallback callback) { - try { - CalculatedFieldTelemetryUpdateRequest request = fromProto(proto.getMsg()); - - if (proto.getLinksList().isEmpty()) { - onTelemetryUpdate(proto.getMsg(), callback); - return; - } - - proto.getLinksList().forEach(ctxIdProto -> { - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId); - - Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); - if (!updatedTelemetry.isEmpty()) { - EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); - } - }); - } catch (Exception e) { - log.trace("Failed to process telemetry update msg: [{}]", proto, e); - } - } - - private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); - Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - -// updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - - @Override - public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - } - if (proto.getAdded()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - if (proto.getUpdated()) { - log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); - - EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); - EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - - calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); - initializeStateForEntityByProfile(entityId, newProfileId, callback); - } - } else { - clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); - } - } catch (Exception e) { - log.trace("Failed to process entity update msg: [{}]", proto, e); - } - } +// @Override +// public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { +// try { +// TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); +// +// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); +// if (tpi.isMyPartition()) { +// log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); +// if (proto.getDeleted()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// } +// if (proto.getAdded()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// if (proto.getUpdated()) { +// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); +// +// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); +// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); +// +// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); +// initializeStateForEntityByProfile(entityId, newProfileId, callback); +// } +// } else { +// clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); +// } +// } catch (Exception e) { +// log.trace("Failed to process entity update msg: [{}]", proto, e); +// } +// } private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..d37e7ebc3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -28,6 +28,6 @@ public interface CalculatedFieldStateService { void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - void removeState(CalculatedFieldEntityCtxId ctxId); + void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 37e402fbc8..6ca4369230 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import java.util.ArrayList; @@ -154,4 +153,12 @@ public class CalculatedFieldCtx { public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } + + public boolean hasSignificantChanges(CalculatedFieldCtx other) { + boolean entityIdChanged = !entityId.equals(other.entityId); + boolean typeChanged = !cfType.equals(other.cfType); + boolean argumentsChanged = !arguments.equals(other.arguments); + return entityIdChanged || typeChanged || argumentsChanged; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..f5c882ea52 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -59,8 +58,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId) { + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + callback.onSuccess(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 95b340c362..d40bf7b130 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -56,6 +56,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.TbQueueCallback; import java.util.Set; @@ -124,7 +125,7 @@ public class EntityStateSourcingListener { tbClusterService.onApiStateChange(apiUsageState, null); } case CALCULATED_FIELD -> { - onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity(), lifecycleEvent); + onCalculatedFieldUpdate(event.getEntity(), event.getOldEntity()); } default -> { } @@ -193,8 +194,7 @@ public class EntityStateSourcingListener { } case CALCULATED_FIELD -> { CalculatedField calculatedField = (CalculatedField) event.getEntity(); - ComponentLifecycleMsg lifecycleMsg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); - tbClusterService.onCalculatedFieldDeleted(calculatedField.getTenantId(), calculatedField, lifecycleMsg); + tbClusterService.onCalculatedFieldDeleted(calculatedField, null); } default -> { } @@ -276,13 +276,13 @@ public class EntityStateSourcingListener { } } - private void onCalculatedFieldUpdate(Object entity, Object oldEntity, ComponentLifecycleEvent lifecycleEvent) { + private void onCalculatedFieldUpdate(Object entity, Object oldEntity) { CalculatedField calculatedField = (CalculatedField) entity; CalculatedField oldCalculatedField = null; if (oldEntity instanceof CalculatedField) { oldCalculatedField = (CalculatedField) oldEntity; } - tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), lifecycleEvent)); + tbClusterService.onCalculatedFieldUpdated(calculatedField, oldCalculatedField, TbQueueCallback.EMPTY); } private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 0fcc04bd5d..3d8bbddc80 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; @@ -25,20 +24,17 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; +import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -70,6 +66,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.thingsboard.server.common.util.ProtoUtils.fromProto; + @Service @TbRuleEngineComponent @Slf4j @@ -86,8 +84,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private final TbRuleEngineQueueFactory queueFactory; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; - private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; private volatile ListeningExecutorService calculatedFieldsExecutor; @@ -101,12 +97,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldExecutionService calculatedFieldExecutionService, CalculatedFieldCache calculatedFieldCache) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; } @PostConstruct @@ -168,12 +162,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer forwardToActorSystem(toCfMsg.getLinkedTelemetryMsg(), callback); } else if (toCfMsg.hasComponentLifecycleMsg()) { log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfMsg.getComponentLifecycleMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getComponentLifecycleMsg(), callback); - } else if (toCfMsg.hasEntityUpdateMsg()) { - log.trace("[{}] Forwarding entity update message for processing {}", id, toCfMsg.getEntityUpdateMsg()); - /// TODO: forward to Actor system - forwardToCalculatedFieldService(toCfMsg.getEntityUpdateMsg(), callback); + forwardToActorSystem(toCfMsg.getComponentLifecycleMsg(), callback); } } catch (Throwable e) { log.warn("[{}] Failed to process message: {}", id, msg, e); @@ -222,14 +211,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void handleNotification(UUID id, TbProtoQueueMsg msg, TbCallback callback) { ToCalculatedFieldNotificationMsg toCfNotification = msg.getValue(); - if (toCfNotification.hasComponentLifecycle()) { + if (toCfNotification.hasComponentLifecycleMsg()) { // from upstream (maybe removed since we don't need to init state for each partition) - forwardToActorSystem(toCfNotification.getComponentLifecycle(), callback); - handleComponentLifecycleMsg(id, ProtoUtils.fromProto(toCfNotification.getComponentLifecycle())); - } else if (toCfNotification.hasEntityUpdateMsg()) { - processEntityUpdateMsg(toCfNotification.getEntityUpdateMsg()); - // from upstream (maybe removed since we don't need to update state for each partition) - forwardToActorSystem(toCfNotification.getEntityUpdateMsg(), callback); + log.trace("[{}] Forwarding component lifecycle message for processing {}", id, toCfNotification.getComponentLifecycleMsg()); + forwardToActorSystem(toCfNotification.getComponentLifecycleMsg(), callback); } else if (toCfNotification.hasLinkedTelemetryMsg()) { forwardToActorSystem(toCfNotification.getLinkedTelemetryMsg(), callback); } @@ -248,74 +233,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer actorContext.tell(new CalculatedFieldLinkedTelemetryMsg(tenantId, entityId, linkedMsg, callback)); } - private void forwardToCalculatedFieldService(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(ComponentLifecycleMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var calculatedFieldId = new CalculatedFieldId(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldLifecycleMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field message for calculated field [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToCalculatedFieldService(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void forwardToActorSystem(CalculatedFieldEntityUpdateMsgProto msg, TbCallback callback) { - var tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(msg.getEntityType(), new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); - ListenableFuture future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityUpdateMsg(msg, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process entity updated message for entity [{}]", tenantId.getId(), entityId.getId(), t); - callback.onFailure(t); - }); - } - - private void processEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto entityUpdateMsg) { - var tenantId = toTenantId(entityUpdateMsg.getTenantIdMSB(), entityUpdateMsg.getTenantIdLSB()); - var entityId = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityType(), new UUID(entityUpdateMsg.getEntityIdMSB(), entityUpdateMsg.getEntityIdLSB())); - if (entityUpdateMsg.getAdded()) { - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } else if (entityUpdateMsg.getDeleted()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - } else if (entityUpdateMsg.getUpdated()) { - var oldProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getOldProfileIdMSB(), entityUpdateMsg.getOldProfileIdLSB())); - var newProfile = EntityIdFactory.getByTypeAndUuid(entityUpdateMsg.getEntityProfileType(), new UUID(entityUpdateMsg.getNewProfileIdMSB(), entityUpdateMsg.getNewProfileIdLSB())); - calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfile).remove(entityId); - calculatedFieldCache.getEntitiesByProfile(tenantId, newProfile).add(entityId); - } - } - - private void throwNotHandled(Object msg, TbCallback callback) { - log.warn("Message not handled: {}", msg); - callback.onFailure(new RuntimeException("Message not handled!")); + private void forwardToActorSystem(ComponentLifecycleMsgProto proto, TbCallback callback) { + var msg = fromProto(proto); + actorContext.tell(new CalculatedFieldEntityLifecycleMsg(msg.getTenantId(), msg, callback)); } private TenantId toTenantId(long tenantIdMSB, long tenantIdLSB) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2783e826fe..b272694b16 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -70,7 +70,6 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto; @@ -362,8 +361,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); - producerProvider.getCalculatedFieldsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); - toRuleEngineMsgs.incrementAndGet(); + pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @Override @@ -435,7 +433,7 @@ public class DefaultTbClusterService implements TbClusterService { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { DeviceId deviceId = device.getId(); gatewayNotificationsService.onDeviceDeleted(device); - handleCalculatedFieldEntityDeleted(tenantId, deviceId, device.getDeviceProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, deviceId); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); @@ -444,7 +442,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { AssetId assetId = asset.getId(); - handleCalculatedFieldEntityDeleted(tenantId, assetId, asset.getAssetProfileId()); + handleCalculatedFieldEntityDeleted(tenantId, assetId); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); } @@ -597,9 +595,7 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { ComponentLifecycleMsgProto componentLifecycleMsgProto = toProto(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) @@ -622,14 +618,6 @@ public class DefaultTbClusterService implements TbClusterService { // No need to push notifications twice tbRuleEngineServices.removeAll(tbCoreServices); } - if (entityType.equals(EntityType.CALCULATED_FIELD)) { - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } for (String serviceId : tbRuleEngineServices) { TopicPartitionInfo tpi = topicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId); ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycle(componentLifecycleMsgProto).build(); @@ -669,56 +657,86 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceUpdated(Device device, Device old) { + public void onDeviceUpdated(Device entity, Device old) { var created = old == null; - broadcastEntityChangeToTransport(device.getTenantId(), device.getId(), device, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean deviceNameChanged = !device.getName().equals(old.getName()); + boolean deviceNameChanged = !entity.getName().equals(old.getName()); if (deviceNameChanged) { - gatewayNotificationsService.onDeviceUpdated(device, old); + gatewayNotificationsService.onDeviceUpdated(entity, old); } - boolean deviceProfileChanged = !device.getDeviceProfileId().equals(old.getDeviceProfileId()); + boolean deviceProfileChanged = !entity.getDeviceProfileId().equals(old.getDeviceProfileId()); if (deviceProfileChanged) { - handleCalculatedFieldEntityUpdated(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getDeviceProfileId()) + .profileId(entity.getDeviceProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { - pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); + pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); } } else { - handleCalculatedFieldEntityAdded(device.getTenantId(), device.getId(), device.getDeviceProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getDeviceProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); - otaPackageStateService.update(device, old); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); + otaPackageStateService.update(entity, old); } @Override - public void onAssetUpdated(Asset asset, Asset old) { + public void onAssetUpdated(Asset entity, Asset old) { var created = old == null; - broadcastEntityChangeToTransport(asset.getTenantId(), asset.getId(), asset, null); + broadcastEntityChangeToTransport(entity.getTenantId(), entity.getId(), entity, null); if (old != null) { - boolean assetTypeChanged = !asset.getType().equals(old.getType()); + boolean assetTypeChanged = !entity.getAssetProfileId().equals(old.getAssetProfileId()); if (assetTypeChanged) { - handleCalculatedFieldEntityUpdated(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.UPDATED) + .oldProfileId(old.getAssetProfileId()) + .profileId(entity.getAssetProfileId()) + .oldName(old.getName()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { - handleCalculatedFieldEntityAdded(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); + ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() + .tenantId(entity.getTenantId()) + .entityId(entity.getId()) + .event(ComponentLifecycleEvent.CREATED) + .profileId(entity.getAssetProfileId()) + .name(entity.getName()) + .build(); + pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @Override - public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg) { - var created = oldCalculatedField == null; - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(calculatedField.getTenantId(), calculatedField.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override - public void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg) { - CalculatedFieldId calculatedFieldId = calculatedField.getId(); - calculatedFieldExecutionService.pushCalculatedFieldLifecycleMsgToQueue(calculatedField, toProto(lifecycleMsg)); - broadcastEntityStateChangeEvent(tenantId, calculatedFieldId, ComponentLifecycleEvent.DELETED); + public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override @@ -849,54 +867,8 @@ public class DefaultTbClusterService implements TbClusterService { } } - private void handleCalculatedFieldEntityAdded(TenantId tenantId, EntityId entityId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, null, newProfileId, true, false, false); + private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { + ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); + pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } - - private void handleCalculatedFieldEntityUpdated(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, newProfileId, false, true, true); - } - - private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId, EntityId oldProfileId) { - handleCalculatedFieldEntityUpdateEvent(tenantId, entityId, oldProfileId, null, false, false, true); - } - - private void handleCalculatedFieldEntityUpdateEvent(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId, boolean added, boolean updated, boolean deleted) { - CalculatedFieldEntityUpdateMsgProto.Builder builder = CalculatedFieldEntityUpdateMsgProto.newBuilder(); - builder.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - builder.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - builder.setEntityType(entityId.getEntityType().name()); - builder.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - builder.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - if (oldProfileId != null) { - builder.setEntityProfileType(oldProfileId.getEntityType().name()); - builder.setOldProfileIdMSB(oldProfileId.getId().getMostSignificantBits()); - builder.setOldProfileIdLSB(oldProfileId.getId().getLeastSignificantBits()); - } - if (newProfileId != null) { - builder.setEntityProfileType(newProfileId.getEntityType().name()); - builder.setNewProfileIdMSB(newProfileId.getId().getMostSignificantBits()); - builder.setNewProfileIdLSB(newProfileId.getId().getLeastSignificantBits()); - } - builder.setAdded(added); - builder.setUpdated(updated); - builder.setDeleted(deleted); - CalculatedFieldEntityUpdateMsgProto msg = builder.build(); - - broadcastEntityUpdateEvent(msg); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setEntityUpdateMsg(msg).build(), null); - } - - private void broadcastEntityUpdateEvent(CalculatedFieldEntityUpdateMsgProto proto) { - TbQueueProducer> toCalculatedFieldProducer = producerProvider.getCalculatedFieldsNotificationsMsgProducer(); - Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); - Set tbCalculatedFieldServices = new HashSet<>(tbRuleEngineServices); - for (String serviceId : tbCalculatedFieldServices) { - TopicPartitionInfo tpi = topicService.getCalculatedFieldNotificationsTopic(serviceId); - ToCalculatedFieldNotificationMsg toCfNotificationMsg = ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(); - toCalculatedFieldProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), toCfNotificationMsg), null); - toRuleEngineNfs.incrementAndGet(); // TODO: add separate counter when we will have new ServiceType.CALCULATED_FIELDS - } - } - } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 588b135891..3d81de1cc4 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -131,8 +131,8 @@ public interface TbClusterService extends TbQueueClusterService { void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId sourceEdgeId); - void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback); - void onCalculatedFieldDeleted(TenantId tenantId, CalculatedField calculatedField, ComponentLifecycleMsg lifecycleMsg); + void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback); } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java index 7943c9d43d..9e8f8e4a0a 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueCallback.java @@ -15,8 +15,22 @@ */ package org.thingsboard.server.queue; + public interface TbQueueCallback { + TbQueueCallback EMPTY = new TbQueueCallback() { + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + + } + + @Override + public void onFailure(Throwable t) { + + } + }; + void onSuccess(TbQueueMsgMetadata metadata); void onFailure(Throwable t); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 91419bee9c..e13e2d2ecb 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -135,14 +135,18 @@ public enum MsgType { EDGE_SYNC_REQUEST_TO_EDGE_SESSION_MSG, EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; - CF_STATE_RESTORE_MSG,// Sent to init particular calculated field entity state; - CF_TELEMETRY_MSG, + CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; + CF_TELEMETRY_MSG, // Sent from queue to actor system; + CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; + + /* CF Manager Actor -> CF Entity actor */ CF_ENTITY_TELEMETRY_MSG, - CF_LINKED_TELEMETRY_MSG, - CF_UPDATE_MSG, - CF_ENTITY_UPDATE_MSG; + CF_ENTITY_INIT_CF_MSG, + CF_ENTITY_DELETE_MSG; @Getter private final boolean ignoreOnStart; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java new file mode 100644 index 0000000000..51eea1f943 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldEntityLifecycleMsg.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; +import org.thingsboard.server.common.msg.queue.TbCallback; + +@Data +public class CalculatedFieldEntityLifecycleMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final ComponentLifecycleMsg data; + private final TbCallback callback; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ENTITY_LIFECYCLE_MSG; + } +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java index 000733cfd8..208b6ab2f5 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleMsg.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.msg.plugin; +import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -37,6 +38,25 @@ public class ComponentLifecycleMsg implements TenantAwareMsg, ToAllNodesMsg { private final TenantId tenantId; private final EntityId entityId; private final ComponentLifecycleEvent event; + private final String oldName; + private final String name; + private final EntityId oldProfileId; + private final EntityId profileId; + + public ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event) { + this(tenantId, entityId, event, null, null, null, null); + } + + @Builder + private ComponentLifecycleMsg(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent event, String oldName, String name, EntityId oldProfileId, EntityId profileId) { + this.tenantId = tenantId; + this.entityId = entityId; + this.event = event; + this.oldName = oldName; + this.name = name; + this.oldProfileId = oldProfileId; + this.profileId = profileId; + } public Optional getRuleChainId() { return entityId.getEntityType() == EntityType.RULE_CHAIN ? Optional.of((RuleChainId) entityId) : Optional.empty(); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..f0a34de08e 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -114,14 +114,28 @@ public class ProtoUtils { } public static TransportProtos.ComponentLifecycleMsgProto toProto(ComponentLifecycleMsg msg) { - return TransportProtos.ComponentLifecycleMsgProto.newBuilder() + var builder = TransportProtos.ComponentLifecycleMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setEntityType(toProto(msg.getEntityId().getEntityType())) .setEntityIdMSB(msg.getEntityId().getId().getMostSignificantBits()) .setEntityIdLSB(msg.getEntityId().getId().getLeastSignificantBits()) - .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())) - .build(); + .setEvent(TransportProtos.ComponentLifecycleEvent.forNumber(msg.getEvent().ordinal())); + if (msg.getProfileId() != null) { + builder.setProfileIdMSB(msg.getProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); + } + if (msg.getOldProfileId() != null) { + builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + } + if (msg.getName() != null) { + builder.setName(msg.getName()); + } + if (msg.getOldName() != null) { + builder.setName(msg.getOldName()); + } + return builder.build(); } public static TransportProtos.EntityTypeProto toProto(EntityType entityType) { @@ -129,11 +143,22 @@ public class ProtoUtils { } public static ComponentLifecycleMsg fromProto(TransportProtos.ComponentLifecycleMsgProto proto) { - return new ComponentLifecycleMsg( - TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), - EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())), - ComponentLifecycleEvent.values()[proto.getEventValue()] - ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(fromProto(proto.getEntityType()), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + var builder = ComponentLifecycleMsg.builder() + .tenantId(TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()))) + .entityId(entityId) + .event(ComponentLifecycleEvent.values()[proto.getEventValue()]) + .name(proto.getName()) + .oldName(proto.getOldName()); + if (proto.getProfileIdMSB() != 0 || proto.getProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.profileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB()))); + } + if (proto.getOldProfileIdMSB() != 0 || proto.getOldProfileIdLSB() != 0) { + var profileType = EntityType.DEVICE.equals(entityId.getEntityType()) ? EntityType.DEVICE_PROFILE : EntityType.ASSET_PROFILE; + builder.oldProfileId(EntityIdFactory.getByTypeAndUuid(profileType, new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB()))); + } + return builder.build(); } public static EntityType fromProto(TransportProtos.EntityTypeProto entityType) { diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 69b912120e..18bcb7c6f4 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -773,22 +773,6 @@ message DeviceInactivityProto { int64 lastInactivityTime = 5; } -message CalculatedFieldEntityUpdateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; - string entityProfileType = 6; - int64 oldProfileIdMSB = 7; - int64 oldProfileIdLSB = 8; - int64 newProfileIdMSB = 9; - int64 newProfileIdLSB = 10; - bool added = 11; - bool updated = 12; - bool deleted = 13; -} - message CalculatedFieldTelemetryMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -1203,6 +1187,13 @@ message ComponentLifecycleMsgProto { int64 entityIdMSB = 4; int64 entityIdLSB = 5; ComponentLifecycleEvent event = 6; + //Since 4.0. To replace the + string oldName = 7; + string name = 8; + int64 oldProfileIdMSB = 9; + int64 oldProfileIdLSB = 10; + int64 profileIdMSB = 11; + int64 profileIdLSB = 12; } message EdgeEventMsgProto { @@ -1630,16 +1621,14 @@ message ToEdgeEventNotificationMsg { } message ToCalculatedFieldMsg { - CalculatedFieldTelemetryMsgProto telemetryMsg = 1; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; - ComponentLifecycleMsgProto componentLifecycleMsg = 3; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 4; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldTelemetryMsgProto telemetryMsg = 2; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; } message ToCalculatedFieldNotificationMsg { - ComponentLifecycleMsgProto componentLifecycle = 1; - CalculatedFieldEntityUpdateMsgProto entityUpdateMsg = 2; - CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 3; + ComponentLifecycleMsgProto componentLifecycleMsg = 1; + CalculatedFieldLinkedTelemetryMsgProto linkedTelemetryMsg = 2; } /* Messages that are handled by ThingsBoard RuleEngine Service */ From bc835eb9d467f3bcff9777e86944171cdf64a852 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 15:33:30 +0200 Subject: [PATCH 112/438] Fixes --- .../calculated-fields-table-config.ts | 12 +- ...culated-field-arguments-table.component.ts | 2 +- .../calculated-field-dialog.component.ts | 23 +++- ...ulated-field-argument-panel.component.html | 122 +++++++++--------- ...ulated-field-argument-panel.component.scss | 4 + ...lculated-field-argument-panel.component.ts | 11 +- .../entity/entity-autocomplete.component.html | 2 +- .../entity-key-autocomplete.component.html | 12 +- 8 files changed, 111 insertions(+), 77 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ae025a13f6..fcef11799d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -77,12 +77,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig('name', 'common.name', '33%')); - this.columns.push( - new EntityTableColumn('type', 'common.type', '50px')); - this.columns.push( - new EntityTableColumn('expression', 'calculated-fields.expression', '50%', entity => entity.configuration.expression)); + const expressionColumn = new EntityTableColumn('expression', 'calculated-fields.expression', '33%', entity => entity.configuration?.expression); + expressionColumn.sortable = false; + + this.columns.push(new EntityTableColumn('name', 'common.name', '33%')); + this.columns.push(new EntityTableColumn('type', 'common.type', '50px')); + this.columns.push(expressionColumn); this.cellActionDescriptors.push( { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 1dedc5e80e..657b20abb4 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -133,7 +133,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }; this.keysPopupClosed = false; const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'leftBottom', false, null, + this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 7f9af5cead..71fea36959 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -84,10 +84,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); - this.toggleScopeByOutputType(this.outputFormGroup.get('type').value); + this.observeTypeChanges(); } get configFormGroup(): FormGroup { @@ -113,10 +110,26 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.toggleScopeByOutputType(type)); + this.fieldFormGroup.get('type').valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(type => this.toggleKeyByCalculatedFieldType(type)); } private toggleScopeByOutputType(type: OutputType): void { this.outputFormGroup.get('scope')[type === OutputType.Attribute? 'enable' : 'disable']({emitEvent: false}); } + + private toggleKeyByCalculatedFieldType(type: CalculatedFieldType): void { + this.outputFormGroup.get('name')[type === CalculatedFieldType.SIMPLE? 'enable' : 'disable']({emitEvent: false}); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 865b71406d..58923e0029 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -24,33 +24,35 @@
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - +
+ @if (argumentFormGroup.get('argumentName').touched) { + @if (argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } } - } +
@@ -117,40 +119,42 @@ } - @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { -
-
{{ 'calculated-fields.timeseries-key' | translate }}
- -
- } @else { -
-
{{ 'calculated-fields.attribute-scope' | translate }}
- - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if ((keyEntityType$ | async) === EntityType.DEVICE) { - - {{ 'calculated-fields.client-attributes' | translate }} - - - {{ 'calculated-fields.shared-attributes' | translate }} + @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) { +
+
{{ 'calculated-fields.timeseries-key' | translate }}
+ +
+ } @else { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} - } - - -
-
-
{{ 'calculated-fields.attribute-key' | translate }}
- -
+ @if ((keyEntityType$ | async) === EntityType.DEVICE) { + + {{ 'calculated-fields.client-attributes' | translate }} + + + {{ 'calculated-fields.shared-attributes' | translate }} + + } + + +
+
+
{{ 'calculated-fields.attribute-key' | translate }}
+ +
+ } } @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Rolling) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index a784909b92..520f2d3ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -25,5 +25,9 @@ width: 100%; } } + + .mat-mdc-form-field-infix { + display: flex; + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b476bc0fee..79f34d7c20 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; +import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewChild } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { debounceTime, delay, distinctUntilChanged, filter, map, startWith } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -92,6 +92,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme constructor( private fb: FormBuilder, + private cd: ChangeDetectorRef ) { super(); @@ -154,22 +155,24 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme default: entityId = this.argumentFormGroup.get('refEntityId').value as any; } - if (onInit) { + if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } this.entityFilter = { type: AliasFilterType.singleEntity, singleEntity: entityId, }; + this.cd.markForCheck(); } private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, + this.refEntityKeyFormGroup.get('type').valueChanges, this.refEntityIdFormGroup.get('id').valueChanges.pipe(filter(Boolean)), this.refEntityKeyFormGroup.get('scope').valueChanges, ) - .pipe(debounceTime(300), delay(50), takeUntilDestroyed()) + .pipe(throttleTime(100), delay(50), takeUntilDestroyed()) .subscribe(() => this.updateEntityFilter(this.entityType)); } diff --git a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html index 44a218b455..cbec73e64d 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-autocomplete.component.html @@ -29,7 +29,7 @@ {{ displayEntityFn(selectEntityFormGroup.get('entity').value) }} - - close + } @else if (keyControl.hasError('required') && keyControl.touched) { + + warning + } @for (key of filteredKeys$ | async; track key) { + } @empty { + {{ 'entity.no-keys-found' | translate }} } From 75369604a1cc1410b9af6249a3eb448d162a416a Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:46:28 +0200 Subject: [PATCH 113/438] Fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../calculated-field-arguments-table.component.ts | 12 ++++++++---- .../dialog/calculated-field-dialog.component.ts | 4 +++- .../calculated-field-argument-panel.component.html | 3 ++- .../calculated-field-argument-panel.component.ts | 13 ++++--------- ui-ngx/src/app/shared/models/regex.constants.ts | 2 ++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 021bd15c58..f6b82dd954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -99,7 +99,7 @@ {{ 'calculated-fields.no-arguments' | translate }} } - @if (errorText) { + @if (errorText && this.argumentsFormArray.dirty) { } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 657b20abb4..7357912bdc 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -98,7 +98,11 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => this.calculatedFieldType() && this.argumentsFormArray.updateValueAndValidity()); + effect(() => { + if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { + this.argumentsFormArray.updateValueAndValidity(); + } + }); } registerOnChange(fn: (argumentsObj: Record) => void): void { @@ -183,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -202,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(noLeadTrailSpacesRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 71fea36959..d30a90e954 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -110,7 +110,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent {{ 'calculated-fields.server-attributes' | translate }} - @if ((keyEntityType$ | async) === EntityType.DEVICE) { + @if (entityType === ArgumentEntityType.Device + || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.client-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 79f34d7c20..2949ffdaee 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -27,7 +27,7 @@ import { CalculatedFieldArgumentValue, CalculatedFieldType } from '@shared/models/calculated-field.models'; -import { delay, distinctUntilChanged, filter, map, startWith, throttleTime } from 'rxjs/operators'; +import { delay, distinctUntilChanged, filter, throttleTime } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; import { AttributeScope, DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { DatasourceType } from '@shared/models/widget.models'; @@ -57,7 +57,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(noLeadTrailSpacesRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -74,11 +74,6 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentTypes: ArgumentType[]; entityFilter: EntityFilter; - keyEntityType$ = this.refEntityIdFormGroup.get('entityType').valueChanges - .pipe( - startWith(this.refEntityIdFormGroup.get('entityType').value), - map(type => type === ArgumentEntityType.Current ? this.entityId.entityType : type) - ); readonly argumentEntityTypes = Object.values(ArgumentEntityType) as ArgumentEntityType[]; readonly ArgumentEntityTypeTranslations = ArgumentEntityTypeTranslations; @@ -140,7 +135,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme this.argumentFormGroup.get('defaultValue')[isRolling? 'disable' : 'enable']({ emitEvent: false }); } - private updateEntityFilter(entityType: ArgumentEntityType, onInit = false): void { + private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { let entityId: EntityId; switch (entityType) { case ArgumentEntityType.Current: diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 55742cefd4..60bf154423 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -15,3 +15,5 @@ /// export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; + +export const charNumRegex = /^[a-zA-Z0-9]+$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index be31b4f70b..ab696e226c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1030,7 +1030,7 @@ "timeseries-key": "Time series key", "device-name": "Device name", "latest-telemetry": "Latest telemetry", - "rolling": "Rolling", + "rolling": "Time series rolling", "attribute-scope": "Attribute scope", "server-attributes": "Server attributes", "client-attributes": "Client attributes", From ff8a9309f4a0cba877aa5d401a325021ed1d2cf6 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 16:57:46 +0200 Subject: [PATCH 114/438] Fixes --- .../calculated-field-arguments-table.component.html | 4 ++-- .../panel/calculated-field-argument-panel.component.ts | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index f6b82dd954..4f1d1d3980 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -28,7 +28,7 @@ - @if (group.get('refEntityId')?.get('id').value) { + @if (group.get('refEntityId')?.get('id')?.value) { @@ -51,7 +51,7 @@ {{ - (group.get('refEntityId')?.get('entityType').value === ArgumentEntityType.Tenant + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant ? 'calculated-fields.argument-current-tenant' : 'calculated-fields.argument-current') | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 2949ffdaee..6e232bdb3d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -119,7 +119,9 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme } saveArgument(): void { - this.argumentsDataApplied.emit({ value: this.argumentFormGroup.value as CalculatedFieldArgumentValue, index: this.index }); + const { refEntityId, ...restConfig } = this.argumentFormGroup.value; + const value = (refEntityId.entityType === ArgumentEntityType.Current ? restConfig : { refEntityId, ...restConfig }) as CalculatedFieldArgumentValue; + this.argumentsDataApplied.emit({ value, index: this.index }); } cancel(): void { @@ -148,7 +150,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as any; + entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); From 6905cb530b915898cf90201bde255a3507c20293 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 31 Jan 2025 17:12:02 +0200 Subject: [PATCH 115/438] added new protos for states --- ...efaultCalculatedFieldExecutionService.java | 23 +-- .../server/service/cf/RocksDBService.java | 28 +++- .../cf/ctx/CalculatedFieldStateService.java | 2 - .../cf/ctx/state/RocksDBStateService.java | 149 ++++++++++++++++-- .../ctx/state/ScriptCalculatedFieldState.java | 3 +- .../ctx/state/SimpleCalculatedFieldState.java | 4 +- .../ctx/state/SingleValueArgumentEntry.java | 14 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 28 +++- .../ctx/state/TsRollingArgumentEntryTest.java | 25 +-- .../server/common/util/ProtoUtils.java | 39 ++++- common/proto/src/main/proto/queue.proto | 25 ++- 11 files changed, 263 insertions(+), 77 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 8f948aa265..7367eb611f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -71,11 +71,13 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityUpdateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleEvent; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; @@ -84,7 +86,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -383,7 +384,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() .filter(entry -> entry.getValue().getRefEntityId() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback); @@ -551,7 +552,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); + .collect(Collectors.toMap(Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); // updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @@ -677,7 +678,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } @@ -710,7 +711,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldLinkedTelemetryMsgProto buildLinkedTelemetryMsgProto(CalculatedFieldTelemetryMsgProto telemetryProto, List links) { - TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.Builder builder = TransportProtos.CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); + Builder builder = CalculatedFieldLinkedTelemetryMsgProto.newBuilder(); builder.setMsg(telemetryProto); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -719,8 +720,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } //TODO: IM: move to utils; - private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) .setEntityType(ctxId.entityId().getEntityType().name()) @@ -890,8 +893,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return telemetryMsg; } - private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { - return TransportProtos.CalculatedFieldIdProto.newBuilder() + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) .build(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 3aed65eced..5eaeff120a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -22,6 +22,8 @@ import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; @@ -49,6 +51,14 @@ public class RocksDBService { } } + public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { + try { + db.put(writeOptions, key.toByteArray(), value.toByteArray()); + } catch (RocksDBException e) { + log.error("Failed to store data to RocksDB", e); + } + } + public void delete(String key) { try { db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); @@ -67,18 +77,20 @@ public class RocksDBService { } } - public Map getAll() { - Map map = new HashMap<>(); + public Map getAll() { + Map results = new HashMap<>(); try (RocksIterator iterator = db.newIterator()) { for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - String key = new String(iterator.key(), StandardCharsets.UTF_8); - String value = new String(iterator.value(), StandardCharsets.UTF_8); - map.put(key, value); + try { + CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); + CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); + results.put(key, value); + } catch (Exception e) { + log.error("Failed to retrieve data from RocksDB", e); + } } - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); } - return map; + return results; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index 2bf6b7e0f2..4f9a998937 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -24,8 +24,6 @@ public interface CalculatedFieldStateService { Map restoreStates(); - CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId ctxId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index aa82f2fc57..4de15f70bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -19,14 +19,28 @@ import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.RollingArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueProto; import org.thingsboard.server.service.cf.RocksDBService; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import java.util.Map; -import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; import java.util.stream.Collectors; @Service @@ -40,21 +54,14 @@ public class RocksDBStateService implements CalculatedFieldStateService { public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( - entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), - entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldState.class) + entry -> fromProto(entry.getKey()), + entry -> fromProto(entry.getValue()) )); } @Override - public CalculatedFieldState restoreState(CalculatedFieldEntityCtxId ctxId) { - return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) - .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldState.class)) - .orElse(null); - } - - @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback){ - rocksDBService.put(JacksonUtil.writeValueAsString(stateId), JacksonUtil.writeValueAsString(state)); + public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); callback.onSuccess(); } @@ -63,4 +70,120 @@ public class RocksDBStateService implements CalculatedFieldStateService { rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } + private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() + .setId(toProto(stateId)) + .setType(state.getType().name()) + .addAllRequiredArguments(state.getRequiredArguments()); + + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + + return builder.build(); + } + + private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(entry.getTs()); + + if (entry.getVersion() != null) { + singleValueProtoBuilder.setVersion(entry.getVersion()); + } + + KvEntry value = entry.getValue(); + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + return SingleValueArgumentProto.newBuilder() + .setArgName(argName) + .setValue(singleValueProtoBuilder.build()) + .build(); + } + + private RollingArgumentProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + RollingArgumentProto.Builder rollingArgumentProtoBuilder = RollingArgumentProto.newBuilder() + .setArgName(argName); + + entry.getTsRecords().forEach((ts, value) -> { + SingleValueProto.Builder singleValueProtoBuilder = SingleValueProto.newBuilder() + .setTs(ts); + + if (value != null) { + singleValueProtoBuilder.setHasV(true) + .setValue(ProtoUtils.toKeyValueProto(value)); + } + + rollingArgumentProtoBuilder.addValues(singleValueProtoBuilder.build()); + }); + + return rollingArgumentProtoBuilder.build(); + } + + private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(proto.getRequiredArgumentsList()); + case SCRIPT -> new ScriptCalculatedFieldState(proto.getRequiredArgumentsList()); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + SingleValueProto valueProto = proto.getValue(); + BasicKvEntry value = valueProto.getHasV() ? ProtoUtils.fromProto(valueProto.getValue()) : null; + + return new SingleValueArgumentEntry(valueProto.getTs(), value, valueProto.getVersion()); + } + + private TsRollingArgumentEntry fromRollingArgumentProto(RollingArgumentProto proto) { + TreeMap tsRecords = new TreeMap<>(); + + proto.getValuesList().forEach(singleValueProto -> { + BasicKvEntry value = singleValueProto.getHasV() ? ProtoUtils.fromProto(singleValueProto.getValue()) : null; + tsRecords.put(singleValueProto.getTs(), value); + }); + + return new TsRollingArgumentEntry(tsRecords); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index a3c7efb388..4a24d13c93 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -53,7 +54,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { arguments.forEach((key, argumentEntry) -> { if (argumentEntry instanceof TsRollingArgumentEntry tsRollingEntry) { Argument argument = ctx.getArguments().get(key); - TreeMap tsRecords = tsRollingEntry.getTsRecords(); + TreeMap tsRecords = tsRollingEntry.getTsRecords(); if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index 8b8fe6e8c7..d233b60512 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -21,6 +21,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.List; @@ -52,7 +53,8 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - expr.setVariable(entry.getKey(), Double.parseDouble(entry.getValue().getValue().toString())); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 5117fcab0e..8d4d40d39b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.util.ProtoUtils; @@ -33,19 +34,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private Object value; + private BasicKvEntry value; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry).getValue(); + this.value = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -56,7 +57,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = entry.getValue(); + this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -72,11 +73,6 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } - @Override - public Object getValue() { - return value; - } - @Override public ArgumentEntry copy() { return new SingleValueArgumentEntry(this.ts, this.value, this.version); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index b86a51ca03..6c1a772c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -21,11 +21,15 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.math.NumberUtils; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.util.ProtoUtils; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; @Data @NoArgsConstructor @@ -37,10 +41,10 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000; - private TreeMap tsRecords = new TreeMap<>(); + private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue())); + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } /** @@ -58,7 +62,15 @@ public class TsRollingArgumentEntry implements ArgumentEntry { @JsonIgnore @Override public Object getValue() { - return tsRecords; + return tsRecords.entrySet() + .stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().getValue(), + (oldValue, newValue) -> oldValue, + TreeMap::new + )); + } @Override @@ -79,7 +91,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private boolean updateTsRollingEntry(TsRollingArgumentEntry tsRollingEntry) { boolean updated = false; - for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { + for (Map.Entry tsRecordEntry : tsRollingEntry.getTsRecords().entrySet()) { updated |= addTsRecordIfAbsent(tsRecordEntry.getKey(), tsRecordEntry.getValue()); } return updated; @@ -89,7 +101,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); } - private boolean addTsRecordIfAbsent(Long ts, Object value) { + private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { if (!tsRecords.containsKey(ts)) { addTsRecord(ts, value); return true; @@ -97,9 +109,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { return false; } - private void addTsRecord(Long ts, Object value) { - if (NumberUtils.isParsable(value.toString())) { - tsRecords.put(ts, value); + private void addTsRecord(Long ts, KvEntry value) { + if (NumberUtils.isParsable(value.getValue().toString())) { + tsRecords.put(ts, ProtoUtils.basicKvEntryFromKvEntry(value)); if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) { tsRecords.pollFirstEntry(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index b08c5f2a58..9ca242092d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import java.util.Map; import java.util.TreeMap; @@ -32,10 +35,10 @@ public class TsRollingArgumentEntryTest { @BeforeEach void setUp() { - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new DoubleDataEntry("key", 10.0)); + values.put(ts - 30, new DoubleDataEntry("key", 12.0)); + values.put(ts - 20, new DoubleDataEntry("key", 17.0)); entry = new TsRollingArgumentEntry(values); } @@ -47,7 +50,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); @@ -56,7 +59,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, 23, 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); assertThat(entry.updateEntry(newEntry)).isFalse(); } @@ -64,10 +67,10 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16); - values.put(ts - 10, 7); - values.put(ts - 5, 1); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new DoubleDataEntry("key", 16.0)); + values.put(ts - 10, new DoubleDataEntry("key", 7.0)); + values.put(ts - 5, new DoubleDataEntry("key", 1.0)); newEntry.setTsRecords(values); assertThat(entry.updateEntry(newEntry)).isTrue(); @@ -83,7 +86,7 @@ public class TsRollingArgumentEntryTest { @Test void testUpdateEntryWhenValueIsNotNumber() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, "string", 123L); + SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 10, new StringDataEntry("key", "string"), 123L); assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index 7ba11d8ea9..91e3276210 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; @@ -90,7 +91,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueProto; import java.util.ArrayList; import java.util.Arrays; @@ -630,6 +631,42 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static BasicKvEntry basicKvEntryFromProto(TransportProtos.AttributeValueProto proto) { + boolean hasValue = proto.getHasV(); + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, hasValue ? proto.getBoolV() : null); + case LONG_V -> new LongDataEntry(key, hasValue ? proto.getLongV() : null); + case DOUBLE_V -> new DoubleDataEntry(key, hasValue ? proto.getDoubleV() : null); + case STRING_V -> new StringDataEntry(key, hasValue ? proto.getStringV() : null); + case JSON_V -> new JsonDataEntry(key, hasValue ? proto.getJsonV() : null); + default -> null; + }; + } + + public static BasicKvEntry fromProto(KeyValueProto proto) { + String key = proto.getKey(); + return switch (proto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, proto.getBoolV()); + case LONG_V -> new LongDataEntry(key, proto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, proto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, proto.getStringV()); + case JSON_V -> new JsonDataEntry(key, proto.getJsonV()); + default -> null; + }; + } + + public static BasicKvEntry basicKvEntryFromKvEntry(KvEntry kvEntry) { + String key = kvEntry.getKey(); + return switch (kvEntry.getDataType()) { + case BOOLEAN -> new BooleanDataEntry(key, kvEntry.getBooleanValue().orElse(null)); + case LONG -> new LongDataEntry(key, kvEntry.getLongValue().orElse(null)); + case DOUBLE -> new DoubleDataEntry(key, kvEntry.getDoubleValue().orElse(null)); + case STRING -> new StringDataEntry(key, kvEntry.getStrValue().orElse(null)); + case JSON -> new JsonDataEntry(key, kvEntry.getJsonValue().orElse(null)); + }; + } + public static TsKvEntry fromProto(TransportProtos.TsKvProto proto) { TransportProtos.KeyValueProto kvProto = proto.getKv(); String key = kvProto.getKey(); diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 53045fc147..2ed535495a 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -810,11 +810,13 @@ message CalculatedFieldLinkedTelemetryMsgProto { } message CalculatedFieldEntityCtxIdProto { - int64 calculatedFieldIdMSB = 1; - int64 calculatedFieldIdLSB = 2; - string entityType = 3; - int64 entityIdMSB = 4; - int64 entityIdLSB = 5; + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 calculatedFieldIdMSB = 3; + int64 calculatedFieldIdLSB = 4; + string entityType = 5; + int64 entityIdMSB = 6; + int64 entityIdLSB = 7; } message CalculatedFieldIdProto { @@ -825,13 +827,8 @@ message CalculatedFieldIdProto { message SingleValueProto { int64 ts = 1; int64 version = 2; - KeyValueType type = 3; bool has_v = 4; - bool bool_v = 5; - int64 long_v = 6; - double double_v = 7; - string string_v = 8; - string json_v = 9; + KeyValueProto value = 5; } message SingleValueArgumentProto { @@ -847,8 +844,10 @@ message RollingArgumentProto { message CalculatedFieldStateProto { CalculatedFieldEntityCtxIdProto id = 1; // int32 version = 2; - repeated SingleValueArgumentProto singleValueArguments = 3; - repeated RollingArgumentProto rollingValueArguments = 4; + string type = 3; + repeated string requiredArguments = 4; + repeated SingleValueArgumentProto singleValueArguments = 5; + repeated RollingArgumentProto rollingValueArguments = 6; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. From 14feedbfa05b11f1d20dfa0f6fe9c17a7ea03f57 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 18:01:28 +0200 Subject: [PATCH 116/438] Changed UI of timeWindow --- ...ulated-field-argument-panel.component.html | 22 +++++++++---------- ...lculated-field-argument-panel.component.ts | 5 +++-- .../assets/locale/locale.constant-en_US.json | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 7c33cc2eb3..b1d463a2d3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -168,22 +168,20 @@ } @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- - - {{ 'common.suffix.ms' | translate }} - +
+
{{ 'calculated-fields.time-window' | translate }}
+
+
{{ 'calculated-fields.limit' | translate }}
-
- - - -
+
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 6e232bdb3d..354b6b9f58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -36,6 +36,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityFilter } from '@shared/models/query/query.models'; import { AliasFilterType } from '@shared/models/alias.models'; import { merge } from 'rxjs'; +import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', @@ -68,8 +69,8 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [null], - timeWindow: [null], + limit: [10], + timeWindow: [MINUTE * 15], }); argumentTypes: ArgumentType[]; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ab696e226c..f0f0671c5c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1037,7 +1037,7 @@ "shared-attributes": "Shared attributes", "attribute-key": "Attribute key", "default-value": "Default value", - "limit": "Limit", + "limit": "Max values", "time-window": "Time window", "customer-name": "Customer name", "timeseries": "Time series", From 8dca91c909138deb18a5b8d1b6aed332bcad2e2e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:08:56 +0200 Subject: [PATCH 117/438] Fixed device filter bug --- .../calculated-fields/calculated-fields-table-config.ts | 6 +++--- .../calculated-fields-table.component.html | 4 +++- .../calculated-fields/calculated-fields-table.component.ts | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index fcef11799d..58caa759da 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -97,7 +97,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.editCalculatedField(entity) + onAction: (_, entity) => this.editCalculatedField(entity), } ); } @@ -171,7 +171,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig - +@if (calculatedFieldsTableConfig) { + +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4c22ad2896..4fda1cc075 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -16,6 +16,7 @@ import { ChangeDetectionStrategy, + ChangeDetectorRef, Component, DestroyRef, effect, @@ -43,7 +44,7 @@ export class CalculatedFieldsTableComponent { @ViewChild(EntitiesTableComponent, {static: true}) entitiesTable: EntitiesTableComponent; - active = input(); + active = input(); entityId = input(); calculatedFieldsTableConfig: CalculatedFieldsTableConfig; @@ -54,6 +55,7 @@ export class CalculatedFieldsTableComponent { private store: Store, private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, + private cd: ChangeDetectorRef, private destroyRef: DestroyRef) { effect(() => { @@ -68,6 +70,7 @@ export class CalculatedFieldsTableComponent { this.popoverService, this.destroyRef, ); + this.cd.markForCheck(); } }); } From dd8ce35b86f2da4102bb1bfebceada167bc72a99 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:18:57 +0200 Subject: [PATCH 118/438] Fixes --- .../calculated-field-arguments-table.component.ts | 6 +++--- .../dialog/calculated-field-dialog.component.ts | 2 +- .../panel/calculated-field-argument-panel.component.html | 2 +- .../panel/calculated-field-argument-panel.component.ts | 8 ++++---- .../entity/entity-key-autocomplete.component.html | 4 +++- ui-ngx/src/app/shared/models/regex.constants.ts | 2 +- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 7 files changed, 14 insertions(+), 12 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 7357912bdc..c8dae67aec 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -50,7 +50,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; -import { charNumRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex } from '@shared/models/regex.constants'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -187,7 +187,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...argumentsObj[key], ...(argumentsObj[key].refEntityId ? { refEntityId: this.fb.group({ @@ -206,7 +206,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { return this.fb.group({ ...value, - argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charNumRegex)]], + argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], ...(value.refEntityId ? { refEntityId: this.fb.group({ entityType: [{ value: value.refEntityId.entityType, disabled: true }], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d30a90e954..6d0687fe9a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -61,7 +61,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index b1d463a2d3..020fad4fd2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -23,7 +23,7 @@
{{ 'calculated-fields.argument-name' | translate }}
- +
@if (argumentFormGroup.get('argumentName').touched) { @if (argumentFormGroup.get('argumentName').hasError('required')) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 354b6b9f58..792742e5d0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -18,7 +18,7 @@ import { ChangeDetectorRef, Component, ElementRef, Input, OnInit, output, ViewCh import { TbPopoverComponent } from '@shared/components/popover.component'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { charNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; +import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, ArgumentEntityTypeTranslations, @@ -58,7 +58,7 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -66,10 +66,10 @@ export class CalculatedFieldArgumentPanelComponent extends PageComponent impleme refEntityKey: this.fb.group({ type: [ArgumentType.LatestTelemetry, [Validators.required]], key: [''], - scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }], + scope: [{ value: AttributeScope.SERVER_SCOPE, disabled: true }, [Validators.required]], }), defaultValue: ['', [Validators.pattern(noLeadTrailSpacesRegex)]], - limit: [10], + limit: [1000], timeWindow: [MINUTE * 15], }); diff --git a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html index 3e4ff6e90d..f1073738f9 100644 --- a/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/entity/entity-key-autocomplete.component.html @@ -43,7 +43,9 @@ @for (key of filteredKeys$ | async; track key) { } @empty { - {{ 'entity.no-keys-found' | translate }} + @if (!this.keyControl.value) { + {{ 'entity.no-keys-found' | translate }} + } } diff --git a/ui-ngx/src/app/shared/models/regex.constants.ts b/ui-ngx/src/app/shared/models/regex.constants.ts index 60bf154423..c6b231ef6e 100644 --- a/ui-ngx/src/app/shared/models/regex.constants.ts +++ b/ui-ngx/src/app/shared/models/regex.constants.ts @@ -16,4 +16,4 @@ export const noLeadTrailSpacesRegex = /^\S+(?: \S+)*$/; -export const charNumRegex = /^[a-zA-Z0-9]+$/; +export const charsWithNumRegex = /^[a-zA-Z]+[a-zA-Z0-9]*$/; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f0f0671c5c..ee348cb6c3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1048,7 +1048,7 @@ "delete-multiple-title": "Are you sure you want to delete { count, plural, =1 {1 calculated field} other {# calculated fields} }?", "delete-multiple-text": "Be careful, after the confirmation all selected calculated fields will be removed and all related data will become unrecoverable.", "hint": { - "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with rolling type.", + "arguments-simple-with-rolling": "Simple type calculated field should not contain keys with time series rolling type.", "arguments-empty": "Arguments should not be empty.", "expression-required": "Expression is required.", "expression-invalid": "Expression is invalid", From c82b6c8c45291d35bb15b128d7313167dd8a9dda Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 31 Jan 2025 19:20:26 +0200 Subject: [PATCH 119/438] Empty arguments fix --- .../components/dialog/calculated-field-dialog.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 6d0687fe9a..c9f1a22157 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -34,7 +34,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -63,6 +63,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) ); From 1a67769f1c3657bca7e1a8a31e558c7c0e5ac0e5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:01:06 +0200 Subject: [PATCH 120/438] added api limits and fixed tests --- ...CalculatedFieldEntityMessageProcessor.java | 28 ++++++------ ...alculatedFieldManagerMessageProcessor.java | 14 ++++++ ...efaultCalculatedFieldExecutionService.java | 8 ++-- .../cf/ctx/state/CalculatedFieldCtx.java | 3 +- .../cf/ctx/state/RocksDBStateService.java | 2 +- .../ctx/state/SimpleCalculatedFieldState.java | 2 +- .../ctx/state/SingleValueArgumentEntry.java | 20 ++++++--- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../cf/DefaultTbCalculatedFieldService.java | 18 -------- .../queue/DefaultTbClusterService.java | 6 +-- .../state/ScriptCalculatedFieldStateTest.java | 43 ++++++++++--------- .../state/SimpleCalculatedFieldStateTest.java | 13 +++--- .../state/SingleValueArgumentEntryTest.java | 15 ++++--- .../ctx/state/TsRollingArgumentEntryTest.java | 12 +++--- .../DefaultTenantProfileConfiguration.java | 7 +++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../CalculatedFieldDataValidator.java | 31 +++++++++++++ .../dao/sql/cf/CalculatedFieldRepository.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 19 files changed, 145 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f0064ff459..9392e3c02a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -198,20 +198,6 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return state; } - private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { - if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { - return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); - } - return null; - } - - private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { - if (!proto.getTbMsgType().isEmpty()) { - return TbMsgType.valueOf(proto.getTbMsgType()); - } - return null; - } - @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { if (state.isReady()) { @@ -290,4 +276,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM return cfIds; } + private UUID toTbMsgId(CalculatedFieldTelemetryMsgProto proto) { + if (proto.getTbMsgIdMSB() != 0 && proto.getTbMsgIdLSB() != 0) { + return new UUID(proto.getTbMsgIdMSB(), proto.getTbMsgIdLSB()); + } + return null; + } + + private TbMsgType toTbMsgType(CalculatedFieldTelemetryMsgProto proto) { + if (!proto.getTbMsgType().isEmpty()) { + return TbMsgType.valueOf(proto.getTbMsgType()); + } + return null; + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index a2577a3f27..d5ed8b8aaa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -217,6 +217,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + try { + cfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(cf)) { + systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(cf.getId(), cfCtx); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -257,6 +264,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } initCf(newCfCtx, callback, true); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index b9a18084b1..d8e42c916d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -61,6 +61,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -69,6 +70,7 @@ import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; @@ -142,14 +144,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final TimeseriesService timeseriesService; private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; + private final ApiLimitService apiLimitService; private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; private final ConcurrentMap states = new ConcurrentHashMap<>(); - private static final int MAX_LAST_RECORDS_VALUE = 1024; - private static final Set supportedReferencedEntities = EnumSet.of( EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT ); @@ -560,7 +561,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long currentTime = System.currentTimeMillis(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long startTs = currentTime - timeWindow; - int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); + long maxDataPoints = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + int limit = argument.getLimit() == 0 ? (int) maxDataPoints : argument.getLimit(); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index caeeabea52..a56aed64f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -197,7 +197,8 @@ public class CalculatedFieldCtx { boolean entityIdChanged = !entityId.equals(other.entityId); boolean typeChanged = !cfType.equals(other.cfType); boolean argumentsChanged = !arguments.equals(other.arguments); - return entityIdChanged || typeChanged || argumentsChanged; + boolean expressionChanged = !expression.equals(other.expression); + return entityIdChanged || typeChanged || argumentsChanged || expressionChanged; } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index eaa1b08f5e..bfed563182 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -115,7 +115,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { singleValueProtoBuilder.setVersion(entry.getVersion()); } - KvEntry value = entry.getValue(); + KvEntry value = entry.getKvEntryValue(); if (value != null) { singleValueProtoBuilder.setHasV(true) .setValue(ProtoUtils.toKeyValueProto(value)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java index d233b60512..01cf9cff70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java @@ -53,7 +53,7 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState { for (Map.Entry entry : this.arguments.entrySet()) { try { - BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getValue(); + BasicKvEntry kvEntry = ((SingleValueArgumentEntry) entry.getValue()).getKvEntryValue(); expr.setVariable(entry.getKey(), Double.parseDouble(kvEntry.getValueAsString())); } catch (NumberFormatException e) { throw new IllegalArgumentException("Argument '" + entry.getKey() + "' is not a number."); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 8d4d40d39b..0832e53e5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -34,19 +35,19 @@ public class SingleValueArgumentEntry implements ArgumentEntry { public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0); private long ts; - private BasicKvEntry value; + private BasicKvEntry kvEntryValue; private Long version; public SingleValueArgumentEntry(TsKvProto entry) { this.ts = entry.getTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.fromProto(entry.getKv()); + this.kvEntryValue = ProtoUtils.fromProto(entry.getKv()); } public SingleValueArgumentEntry(AttributeValueProto entry) { this.ts = entry.getLastUpdateTs(); this.version = entry.getVersion(); - this.value = ProtoUtils.basicKvEntryFromProto(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromProto(entry); } public SingleValueArgumentEntry(KvEntry entry) { @@ -57,7 +58,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { this.ts = attributeKvEntry.getLastUpdateTs(); this.version = attributeKvEntry.getVersion(); } - this.value = ProtoUtils.basicKvEntryFromKvEntry(entry); + this.kvEntryValue = ProtoUtils.basicKvEntryFromKvEntry(entry); } /** @@ -65,7 +66,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { * */ private SingleValueArgumentEntry(int ignored) { this.ts = System.currentTimeMillis(); - this.value = null; + this.kvEntryValue = null; } @Override @@ -73,9 +74,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry { return ArgumentEntryType.SINGLE_VALUE; } + @JsonIgnore + public Object getValue() { + return kvEntryValue.getValue(); + } + @Override public ArgumentEntry copy() { - return new SingleValueArgumentEntry(this.ts, this.value, this.version); + return new SingleValueArgumentEntry(this.ts, this.kvEntryValue, this.version); } @Override @@ -88,7 +94,7 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.value = singleValueEntry.getValue(); + this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; return true; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 6c1a772c44..ddae0c3513 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -98,7 +98,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private boolean updateSingleValueEntry(SingleValueArgumentEntry singleValueEntry) { - return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getValue()); + return addTsRecordIfAbsent(singleValueEntry.getTs(), singleValueEntry.getKvEntryValue()); } private boolean addTsRecordIfAbsent(Long ts, KvEntry value) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java index 184092720c..1ccfe2375c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java @@ -48,9 +48,6 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId; @RequiredArgsConstructor public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { - private static final int MAX_ARGUMENT_SIZE = 10; - private static final int MAX_CALCULATED_FIELD_NUMBER = 10; - private final CalculatedFieldService calculatedFieldService; @Override @@ -62,9 +59,7 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp CalculatedField existingCf = calculatedFieldService.findById(tenantId, calculatedField.getId()); checkForEntityChange(existingCf, calculatedField); } - checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId()); - checkArgumentSize(calculatedField.getConfiguration()); checkReferencedEntities(calculatedField.getConfiguration(), user); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); @@ -129,19 +124,6 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp } - private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) { - if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) { - throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE); - } - } - - private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) { - int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size(); - if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) { - throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER); - } - } - private & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { return switch (entityId.getEntityType()) { case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index b272694b16..944e24480e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -43,7 +43,6 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EdgeId; @@ -103,7 +102,6 @@ import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -729,13 +727,13 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } @Override public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getEntityId(), ComponentLifecycleEvent.DELETED); + var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index cbaea6575c..42ff828dbd 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -34,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -51,7 +53,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 43, 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -65,6 +67,7 @@ public class ScriptCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -93,7 +96,7 @@ public class ScriptCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("deviceTemperature", deviceTemperatureArgEntry, "assetHumidity", assetHumidityArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, 41, 349L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(ts, new LongDataEntry("assetHumidity", 41L), 349L); Map newArgs = Map.of("assetHumidity", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -116,17 +119,17 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4);// will not be used for calculation - values.put(ts - 45000, 2);// will not be used for calculation - values.put(ts - 20, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, new LongDataEntry("deviceTemperature", 4L));// will not be used for calculation + values.put(ts - 45000, new LongDataEntry("deviceTemperature", 2L));// will not be used for calculation + values.put(ts - 20, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); @@ -138,19 +141,19 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000);// will not be used - values.put(ts - 18, 0); - values.put(ts - 16, 0); - values.put(ts - 14, 0); - values.put(ts - 12, 0); - values.put(ts - 10, 0); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 1000L));// will not be used + values.put(ts - 18, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 16, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 14, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 12, new LongDataEntry("deviceTemperature", 0L)); + values.put(ts - 10, new LongDataEntry("deviceTemperature", 0L)); argumentEntry.setTsRecords(values); state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); @@ -161,7 +164,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43)); + assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); } @Test @@ -187,10 +190,10 @@ public class ScriptCalculatedFieldStateTest { TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); long ts = System.currentTimeMillis(); - TreeMap values = new TreeMap<>(); - values.put(ts - 40, 10); - values.put(ts - 30, 12); - values.put(ts - 20, 17); + TreeMap values = new TreeMap<>(); + values.put(ts - 40, new LongDataEntry("deviceTemperature", 10L)); + values.put(ts - 30, new LongDataEntry("deviceTemperature", 12L)); + values.put(ts - 20, new LongDataEntry("deviceTemperature", 17L)); argumentEntry.setTsRecords(values); return argumentEntry; diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index 58a981824c..d6b384d85b 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -46,9 +48,9 @@ public class SimpleCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, 11, 145L); - private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, 15, 165L); - private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, 23, 184L); + private final SingleValueArgumentEntry key1ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("key1", 11L), 145L); + private final SingleValueArgumentEntry key2ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 6, new LongDataEntry("key2", 15L), 165L); + private final SingleValueArgumentEntry key3ArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 3, new LongDataEntry("key3", 23L), 184L); private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; @@ -56,6 +58,7 @@ public class SimpleCalculatedFieldStateTest { @BeforeEach void setUp() { ctx = new CalculatedFieldCtx(getCalculatedField(), null); + ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } @@ -88,7 +91,7 @@ public class SimpleCalculatedFieldStateTest { void testUpdateStateWhenUpdateExistingEntry() { state.arguments = new HashMap<>(Map.of("key1", key1ArgEntry)); - SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), 18, 190L); + SingleValueArgumentEntry newArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis(), new LongDataEntry("key1", 18L), 190L); Map newArgs = Map.of("key1", newArgEntry); boolean stateUpdated = state.updateState(newArgs); @@ -130,7 +133,7 @@ public class SimpleCalculatedFieldStateTest { void testPerformCalculationWhenPassedNotNumber() { state.arguments = new HashMap<>(Map.of( "key1", key1ArgEntry, - "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, "string", 124L), + "key2", new SingleValueArgumentEntry(System.currentTimeMillis() - 9, new StringDataEntry("key2", "string"), 124L), "key3", key3ArgEntry )); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 285da0b423..203d7b3d71 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,7 +30,7 @@ public class SingleValueArgumentEntryTest { @BeforeEach void setUp() { - entry = new SingleValueArgumentEntry(ts, 11, 363L); + entry = new SingleValueArgumentEntry(ts, new LongDataEntry("key", 11L), 363L); } @Test @@ -46,26 +47,26 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWithThaSameTs() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, 13, 363L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts, new LongDataEntry("key", 13L), 363L))).isFalse(); } @Test void testUpdateEntryWhenNewVersionIsNull() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, 13, null))).isTrue(); - assertThat(entry.getValue()).isEqualTo(13); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 16, new LongDataEntry("key", 13L), null))).isTrue(); + assertThat(entry.getValue()).isEqualTo(13L); assertThat(entry.getVersion()).isNull(); } @Test void testUpdateEntryWhenNewVersionIsGreaterThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 369L))).isTrue(); - assertThat(entry.getValue()).isEqualTo(18); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 369L))).isTrue(); + assertThat(entry.getValue()).isEqualTo(18L); assertThat(entry.getVersion()).isEqualTo(369L); } @Test void testUpdateEntryWhenNewVersionIsLessThanCurrent() { - assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, 18, 234L))).isFalse(); + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } } \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index 9ca242092d..5a6f5e96e5 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -54,7 +54,7 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(4); - assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23); + assertThat(entry.getTsRecords().get(ts - 10).getValue()).isEqualTo(23.0); } @Test @@ -76,11 +76,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, 10, - ts - 30, 12, - ts - 20, 17, - ts - 10, 7, - ts - 5, 1 + ts - 40, new DoubleDataEntry("key", 10.0), + ts - 30, new DoubleDataEntry("key", 12.0), + ts - 20, new DoubleDataEntry("key", 17.0), + ts - 10, new DoubleDataEntry("key", 7.0), + ts - 5, new DoubleDataEntry("key", 1.0) )); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 6aa9075a79..0fd630fead 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -135,6 +135,12 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private double warnThreshold; + private long maxCalculatedFieldsPerTenant; + private long maxCalculatedFieldsPerEntity; + private long maxArgumentsPerCF; + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + @Override public long getProfileThreshold(ApiUsageRecordKey key) { return switch (key) { @@ -175,6 +181,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura case DASHBOARD -> maxDashboards; case RULE_CHAIN -> maxRuleChains; case EDGE -> maxEdges; + case CALCULATED_FIELD -> maxCalculatedFieldsPerTenant; default -> 0; }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 23a2eae93e..3efb4011ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -43,4 +43,6 @@ public interface CalculatedFieldDao extends Dao { boolean existsByEntityId(TenantId tenantId, EntityId entityId); + long countCFByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java index 80e421f350..db8997ceb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CalculatedFieldDataValidator.java @@ -17,11 +17,15 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.cf.CalculatedFieldDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; @Component public class CalculatedFieldDataValidator extends DataValidator { @@ -29,13 +33,40 @@ public class CalculatedFieldDataValidator extends DataValidator @Autowired private CalculatedFieldDao calculatedFieldDao; + @Autowired + private ApiLimitService apiLimitService; + + @Override + protected void validateCreate(TenantId tenantId, CalculatedField calculatedField) { + validateNumberOfEntitiesPerTenant(tenantId, EntityType.CALCULATED_FIELD); + validateNumberOfCFsPerEntity(tenantId, calculatedField.getEntityId()); + validateNumberOfArgumentsPerCF(tenantId, calculatedField); + } + @Override protected CalculatedField validateUpdate(TenantId tenantId, CalculatedField calculatedField) { CalculatedField old = calculatedFieldDao.findById(calculatedField.getTenantId(), calculatedField.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing calculated field!"); } + validateNumberOfArgumentsPerCF(tenantId, calculatedField); return old; } + private void validateNumberOfCFsPerEntity(TenantId tenantId, EntityId entityId) { + long maxCFsPerEntity = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxCalculatedFieldsPerEntity); + long countCFByEntityId = calculatedFieldDao.countCFByEntityId(tenantId, entityId); + + if (countCFByEntityId == maxCFsPerEntity) { + throw new DataValidationException("Calculated fields per entity limit reached!"); + } + } + + private void validateNumberOfArgumentsPerCF(TenantId tenantId, CalculatedField calculatedField) { + long maxArgumentsPerCF = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxArgumentsPerCF); + if (calculatedField.getConfiguration().getArguments().size() > maxArgumentsPerCF) { + throw new DataValidationException("Calculated field arguments limit reached!"); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index c0118d4f02..2aeca659bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -38,4 +38,6 @@ public interface CalculatedFieldRepository extends JpaRepository removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + long countByTenantIdAndEntityId(UUID tenantId, UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 34bfa27b16..703bdfbf6f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -88,6 +88,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao getEntityClass() { return CalculatedFieldEntity.class; From e62e7fb20be59db8de4cfc7a61fa2c2a6e44ea56 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 3 Feb 2025 12:26:32 +0200 Subject: [PATCH 121/438] implemented linkMatches() method and deletion of state from db --- .../CalculatedFieldManagerMessageProcessor.java | 2 +- .../server/service/cf/RocksDBService.java | 4 ++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 15 +++++++++++++-- .../service/cf/ctx/state/RocksDBStateService.java | 3 +-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index d5ed8b8aaa..b2ff883236 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -122,7 +122,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedFields.containsKey(msg.getId().cfId())) { getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - // TODO: remove state from storage + cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index 5eaeff120a..fe800f61ad 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -59,9 +59,9 @@ public class RocksDBService { } } - public void delete(String key) { + public void delete(CalculatedFieldEntityCtxIdProto key) { try { - db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + db.delete(writeOptions, key.toByteArray()); } catch (RocksDBException e) { log.error("Failed to delete data from RocksDB", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index a56aed64f7..4600628838 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -185,8 +186,18 @@ public class CalculatedFieldCtx { } public boolean linkMatches(EntityId entityId, CalculatedFieldTelemetryMsgProto proto) { - //TODO: IM - implement - return true; + if (!proto.getTsDataList().isEmpty()) { + List updatedTelemetry = proto.getTsDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry); + } else { + AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); + List updatedTelemetry = proto.getAttrDataList().stream() + .map(ProtoUtils::fromProto) + .toList(); + return linkMatches(entityId, updatedTelemetry, scope); + } } public CalculatedFieldEntityCtxId toCalculatedFieldEntityCtxId() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index bfed563182..6487ce1a43 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -67,7 +66,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + rocksDBService.delete(toProto(ctxId)); callback.onSuccess(); } From 0590b5b271c0f464f8ce7ec75f22c669b2ebb979 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 3 Feb 2025 17:54:11 +0200 Subject: [PATCH 122/438] Improvements, refactoring, review comments resolve --- .../calculated-fields-table-config.ts | 36 ++-- .../calculated-fields-table.component.ts | 3 + ...lated-field-arguments-table.component.html | 184 +++++++++--------- ...lated-field-arguments-table.component.scss | 13 +- ...culated-field-arguments-table.component.ts | 44 ++--- .../calculated-field-dialog.component.html | 15 +- .../calculated-field-dialog.component.ts | 32 +-- ...ulated-field-argument-panel.component.html | 131 +++++-------- ...ulated-field-argument-panel.component.scss | 33 ---- ...lculated-field-argument-panel.component.ts | 22 +-- .../entity-debug-settings-button.component.ts | 1 - .../entity-debug-settings-panel.component.ts | 6 +- .../entity/entities-table.component.ts | 4 +- .../entity/entity-table-component.models.ts | 1 - .../pages/device/device-tabs.component.html | 3 +- .../entity/entity-autocomplete.component.html | 12 +- .../entity/entity-autocomplete.component.ts | 4 +- .../entity-key-autocomplete.component.ts | 22 ++- .../shared/models/calculated-field.models.ts | 14 +- ui-ngx/src/app/shared/models/constants.ts | 1 + .../assets/locale/locale.constant-en_US.json | 4 + 21 files changed, 265 insertions(+), 320 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 58caa759da..d8c02558f8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -19,7 +19,7 @@ import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.m import { TranslateService } from '@ngx-translate/core'; import { Direction } from '@shared/models/page/sort-order'; import { MatDialog } from '@angular/material/dialog'; -import { TimePageLink } from '@shared/models/page/page-link'; +import { PageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -27,7 +27,7 @@ import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; -import { DestroyRef } from '@angular/core'; +import { DestroyRef, Renderer2 } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -35,10 +35,10 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDialogComponent } from './components/public-api'; -export class CalculatedFieldsTableConfig extends EntityTableConfig { +export class CalculatedFieldsTableConfig extends EntityTableConfig { // TODO: [Calculated Fields] remove hardcode when BE variable implemented readonly calculatedFieldsDebugPerTenantLimitsConfiguration = @@ -54,20 +54,16 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.fetchCalculatedFields(pageLink); + this.entitiesFetchFunction = (pageLink: PageLink) => this.fetchCalculatedFields(pageLink); this.addEntity = this.addCalculatedField.bind(this); this.deleteEntityTitle = (field: CalculatedField) => this.translate.instant('calculated-fields.delete-title', {title: field.name}); this.deleteEntityContent = () => this.translate.instant('calculated-fields.delete-text'); @@ -102,12 +98,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig> { + fetchCalculatedFields(pageLink: PageLink): Observable> { return this.calculatedFieldsService.getCalculatedFields(this.entityId, pageLink); } onOpenDebugConfig($event: Event, { debugSettings = {}, id }: CalculatedField): void { - const { renderer, viewContainerRef } = this.getTable(); + const { viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); } @@ -115,7 +111,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.onDebugConfigChanged(id.id, settings); debugStrategyPopover.hide(); @@ -133,17 +128,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + return this.getCalculatedFieldDialog() .pipe( filter(Boolean), switchMap(calculatedField => this.calculatedFieldsService.saveCalculatedField({ entityId: this.entityId, ...calculatedField })), ) - .subscribe((res) => { - if (res) { - this.updateData(); - } - }); } private editCalculatedField(calculatedField: CalculatedField): void { @@ -159,8 +149,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { - return this.dialog.open(CalculatedFieldDialogComponent, { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add'): Observable { + return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index 4fda1cc075..bc979a5f0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -21,6 +21,7 @@ import { DestroyRef, effect, input, + Renderer2, ViewChild, } from '@angular/core'; import { EntityId } from '@shared/models/id/entity-id'; @@ -56,6 +57,7 @@ export class CalculatedFieldsTableComponent { private durationLeft: DurationLeftPipe, private popoverService: TbPopoverService, private cd: ChangeDetectorRef, + private renderer: Renderer2, private destroyRef: DestroyRef) { effect(() => { @@ -69,6 +71,7 @@ export class CalculatedFieldsTableComponent { this.durationLeft, this.popoverService, this.destroyRef, + this.renderer ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4f1d1d3980..d1d9998e5a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -15,96 +15,100 @@ limitations under the License. --> -
-
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
-
-
- @for (group of argumentsFormArray.controls; track group) { -
- - - - @if (group.get('refEntityId')?.get('id')?.value) { - - - - - {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} - - - - - - } @else { - - - - {{ - (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant - ? 'calculated-fields.argument-current-tenant' - : 'calculated-fields.argument-current') | translate - }} - - - - } - - - @if (group.get('refEntityKey').get('type').value; as type) { - - - {{ ArgumentTypeTranslations.get(type) | translate }} - - +
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
+
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + +
+ @if (group.get('refEntityId')?.get('id')?.value) { + + + + + {{ entityTypeTranslations.get(group.get('refEntityId').get('entityType').value)?.type | translate }} + + + + + + } @else { + + + + {{ + (group.get('refEntityId')?.get('entityType')?.value === ArgumentEntityType.Tenant + ? 'calculated-fields.argument-current-tenant' + : 'calculated-fields.argument-current') | translate + }} + + + + } +
+ + + @if (group.get('refEntityKey').get('type').value; as type) { + + + {{ ArgumentTypeTranslations.get(type) | translate }} + + + } + + + +
+ {{ group.get('refEntityKey').get('key').value }} +
+ + + +
+ + +
+
+ } @empty { + {{ 'calculated-fields.no-arguments' | translate }} } - - - -
- {{ group.get('refEntityKey').get('key').value }} -
- - - -
- -
-
- } @empty { - {{ 'calculated-fields.no-arguments' | translate }} - } -
- @if (errorText && this.argumentsFormArray.dirty) { - - } -
-
- + @if (errorText && this.argumentsFormArray.dirty) { + + } +
+
+ +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8695ee4068..73f03dc497 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,18 +14,7 @@ * limitations under the License. */ :host ::ng-deep { - .inline-entity-autocomplete { - .mat-mdc-form-field-infix { - padding-top: 8px; - padding-bottom: 8px; - min-height: 40px; - width: auto; - .mdc-text-field__input, .mat-mdc-select { - font-weight: 400; - line-height: 20px; - } - } - + .tb-inline-field { a { font-size: 14px; } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index c8dae67aec..328a82184b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,7 +17,6 @@ import { ChangeDetectorRef, Component, - DestroyRef, effect, forwardRef, input, @@ -29,6 +28,7 @@ import { AbstractControl, ControlValueAccessor, FormBuilder, + FormGroup, NG_VALIDATORS, NG_VALUE_ACCESSOR, ValidationErrors, @@ -51,6 +51,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; import { isDefinedAndNotNull } from '@core/utils'; import { charsWithNumRegex } from '@shared/models/regex.constants'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-calculated-field-arguments-table', @@ -78,20 +79,19 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces errorText = ''; argumentsFormArray = this.fb.array([]); - keysPopupClosed = true; readonly entityTypeTranslations = entityTypeTranslations; readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; constructor( private fb: FormBuilder, private popoverService: TbPopoverService, private viewContainerRef: ViewContainerRef, - private destroyRef: DestroyRef, private cd: ChangeDetectorRef, private renderer: Renderer2 ) { @@ -123,6 +123,9 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces manageArgument($event: Event, matButton: MatButton, index?: number): void { $event?.stopPropagation(); + if (this.popoverComponent && !this.popoverComponent.tbHidden) { + this.popoverComponent.hide(); + } const trigger = matButton._elementRef.nativeElement; if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); @@ -135,27 +138,22 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, }; - this.keysPopupClosed = false; - const argumentsPanelPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, ctx, {}, {}, {}, true); - argumentsPanelPopover.tbComponentRef.instance.popover = argumentsPanelPopover; - argumentsPanelPopover.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { - argumentsPanelPopover.hide(); + this.popoverComponent.tbComponentRef.instance.argumentsDataApplied.subscribe(({ value, index }) => { + this.popoverComponent.hide(); const formGroup = this.getArgumentFormGroup(value); if (isDefinedAndNotNull(index)) { this.argumentsFormArray.setControl(index, formGroup); } else { this.argumentsFormArray.push(formGroup); } - this.argumentsFormArray.markAsDirty(); + formGroup.markAsDirty(); this.cd.markForCheck(); }); - argumentsPanelPopover.tbHideStart.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { - this.keysPopupClosed = true; - }); } } @@ -171,8 +169,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private getArgumentsObject(): Record { - return this.argumentsFormArray.controls.reduce((acc, control) => { - const rawValue = control.getRawValue(); + return this.argumentsFormArray.getRawValue().reduce((acc, rawValue) => { const { argumentName, ...argument } = rawValue as CalculatedFieldArgumentValue; acc[argumentName] = argument; return acc; @@ -186,24 +183,15 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces private populateArgumentsFormArray(argumentsObj: Record): void { Object.keys(argumentsObj).forEach(key => { - this.argumentsFormArray.push(this.fb.group({ - argumentName: [key, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], + const value: CalculatedFieldArgumentValue = { ...argumentsObj[key], - ...(argumentsObj[key].refEntityId ? { - refEntityId: this.fb.group({ - entityType: [{ value: argumentsObj[key].refEntityId.entityType, disabled: true }], - id: [{ value: argumentsObj[key].refEntityId.id , disabled: true }], - }), - } : {}), - refEntityKey: this.fb.group({ - type: [{ value: argumentsObj[key].refEntityKey.type, disabled: true }], - key: [{ value: argumentsObj[key].refEntityKey.key, disabled: true }], - }), - }) as AbstractControl); + argumentName: key + }; + this.argumentsFormArray.push(this.getArgumentFormGroup(value), {emitEvent: false}); }); } - private getArgumentFormGroup(value: CalculatedFieldArgumentValue): AbstractControl { + private getArgumentFormGroup(value: CalculatedFieldArgumentValue): FormGroup { return this.fb.group({ ...value, argumentName: [value.argumentName, [Validators.required, Validators.maxLength(255), Validators.pattern(charsWithNumRegex)]], diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index e6adc1b4d8..ca27ac6fd1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -19,7 +19,7 @@

{{ 'entity.type-calculated-field' | translate}}

-
+
{{ 'common.type' | translate }} - + @for (type of fieldTypes; track type) { {{ CalculatedFieldTypeTranslations.get(type) | translate}} } @@ -102,10 +103,10 @@
{{ 'calculated-fields.output' | translate }}
-
+
{{ 'calculated-fields.output-type' | translate }} - + @for (type of outputTypes; track type) { {{ OutputTypeTranslations.get(type) | translate}} } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c9f1a22157..c8b2073309 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -18,10 +18,9 @@ import { Component, Inject } from '@angular/core'; import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { FormGroup, UntypedFormBuilder, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { helpBaseUrl } from '@shared/models/constants'; import { CalculatedField, CalculatedFieldConfiguration, @@ -35,7 +34,6 @@ import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; -import { isObject } from '@core/utils'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -50,7 +48,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent isObject(configuration?.arguments) ? Object.keys(configuration.arguments) : []) + map(configuration => Object.keys(configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; @@ -73,7 +71,6 @@ export class CalculatedFieldDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDialogData, - public dialogRef: MatDialogRef, - public fb: UntypedFormBuilder) { + protected dialogRef: MatDialogRef, + private fb: FormBuilder) { super(store, router, dialogRef); this.applyDialogData(); this.observeTypeChanges(); @@ -104,14 +101,15 @@ export class CalculatedFieldDialogComponent extends DialogComponent
{{ 'calculated-fields.argument-name' | translate }}
-
- - -
- @if (argumentFormGroup.get('argumentName').touched) { - @if (argumentFormGroup.get('argumentName').hasError('required')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('pattern')) { - - warning - - } @else if (argumentFormGroup.get('argumentName').hasError('maxlength')) { - - warning - - } - } -
- -
+ + + @if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('required')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('maxlength')) { + + warning + + } +
-
{{ 'entity.entity-type' | translate }}
+
{{ 'entity.entity-type' | translate }}
@for (type of argumentEntityTypes; track type) { @@ -67,34 +61,17 @@
- @if (entityType === ArgumentEntityType.Device || entityType === ArgumentEntityType.Asset) { -
-
{{ 'calculated-fields.device-name' | translate }}
- -
- } @else if (entityType === ArgumentEntityType.Customer) { + @if (ArgumentEntityTypeParamsMap.has(entityType)) {
-
{{ 'calculated-fields.customer-name' | translate }}
+
{{ ArgumentEntityTypeParamsMap.get(entityType).title | translate }}
} @@ -123,13 +100,13 @@ @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
- +
} @else {
{{ 'calculated-fields.attribute-scope' | translate }}
- - + + {{ 'calculated-fields.server-attributes' | translate }} @@ -149,7 +126,7 @@
{{ 'calculated-fields.attribute-key' | translate }}
{{ 'calculated-fields.default-value' | translate }}
-
- - - -
+ + +
} @else { -
-
{{ 'calculated-fields.time-window' | translate }}
-
- -
+
+
{{ 'calculated-fields.time-window' | translate }}
+
{{ 'calculated-fields.limit' | translate }}
- +
}
-
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 73f03dc497..8f8b1d5d81 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,4 +19,9 @@ font-size: 14px; } } + .edit-hint { + .mat-mdc-form-field-error { + font-size: 16px; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 328a82184b..a004e3db6f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -21,7 +21,9 @@ import { forwardRef, input, Input, + OnChanges, Renderer2, + SimpleChanges, ViewContainerRef, } from '@angular/core'; import { @@ -70,10 +72,11 @@ import { TbPopoverComponent } from '@shared/components/popover.component'; } ], }) -export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator { +export class CalculatedFieldArgumentsTableComponent implements ControlValueAccessor, Validator, OnChanges { @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; calculatedFieldType = input() @@ -84,6 +87,8 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces readonly ArgumentTypeTranslations = ArgumentTypeTranslations; readonly EntityType = EntityType; readonly ArgumentEntityType = ArgumentEntityType; + readonly ArgumentType = ArgumentType; + readonly CalculatedFieldType = CalculatedFieldType; private popoverComponent: TbPopoverComponent; private propagateChange: (argumentsObj: Record) => void = () => {}; @@ -105,6 +110,13 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }); } + ngOnChanges(changes: SimpleChanges): void { + if (changes.calculatedFieldType?.previousValue + && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { + this.argumentsFormArray.markAsDirty(); + } + } + registerOnChange(fn: (argumentsObj: Record) => void): void { this.propagateChange = fn; } @@ -137,6 +149,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces calculatedFieldType: this.calculatedFieldType(), buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, + entityName: this.entityName, }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -202,6 +215,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces }), } : {}), refEntityKey: this.fb.group({ + ...value.refEntityKey, type: [{ value: value.refEntityKey.type, disabled: true }], key: [{ value: value.refEntityKey.key, disabled: true }], }), diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ca27ac6fd1..37e96cfd91 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -69,6 +69,7 @@ formControlName="arguments" [entityId]="data.entityId" [tenantId]="data.tenantId" + [entityName]="data.entityName" [calculatedFieldType]="fieldFormGroup.get('type').value" />
@@ -114,7 +115,7 @@ @if (outputFormGroup.get('type').value === OutputType.Attribute) { - {{ 'calculated-fields.output-type' | translate }} + {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index c8b2073309..155a384273 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,6 +36,7 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -59,10 +60,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(configuration.arguments)) + startWith(null), + map(() => Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 56568a7bfe..d274c30b8f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -96,7 +96,7 @@ }
- @if (entityFilter.singleEntity.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { + @if (entityFilter.singleEntity?.id || entityType === ArgumentEntityType.Current || entityType === ArgumentEntityType.Tenant) { @if (refEntityKeyFormGroup.get('type').value !== ArgumentType.Attribute) {
{{ 'calculated-fields.timeseries-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 60d79d7bd6..b47d2f073c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -49,6 +49,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() argument: CalculatedFieldArgumentValue; @Input() entityId: EntityId; @Input() tenantId: string; + @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -83,6 +84,8 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { readonly ArgumentEntityType = ArgumentEntityType; readonly ArgumentEntityTypeParamsMap = ArgumentEntityTypeParamsMap; + private currentEntityFilter: EntityFilter; + constructor( private fb: FormBuilder, private cd: ChangeDetectorRef, @@ -107,6 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); + this.currentEntityFilter = this.getCurrentEntityFilter(); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -138,30 +142,53 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { } private updateEntityFilter(entityType: ArgumentEntityType = ArgumentEntityType.Current, onInit = false): void { - let entityId: EntityId; + let entityFilter: EntityFilter; switch (entityType) { case ArgumentEntityType.Current: - entityId = this.entityId + entityFilter = this.currentEntityFilter; break; case ArgumentEntityType.Tenant: - entityId = { - id: this.tenantId, - entityType: EntityType.TENANT + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: { + id: this.tenantId, + entityType: EntityType.TENANT + }, }; break; default: - entityId = this.argumentFormGroup.get('refEntityId').value as unknown as EntityId; + entityFilter = { + type: AliasFilterType.singleEntity, + singleEntity: this.argumentFormGroup.get('refEntityId').value as unknown as EntityId, + }; } if (!onInit) { this.argumentFormGroup.get('refEntityKey').get('key').setValue(''); } - this.entityFilter = { - type: AliasFilterType.singleEntity, - singleEntity: entityId, - }; + this.entityFilter = entityFilter; this.cd.markForCheck(); } + private getCurrentEntityFilter(): EntityFilter { + switch (this.entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [this.entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: this.entityId, + }; + } + } + private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html index 9084301783..f70d4f443b 100644 --- a/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset-profile/asset-profile-tabs.component.html @@ -15,6 +15,10 @@ limitations under the License. --> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 4329cd7e3c..2b677dc278 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -32,6 +32,10 @@ [entityName]="entity.name"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index aa928687ad..37cfafa9aa 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -69,6 +69,10 @@
+ + + diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 533384e1c4..f73926f62b 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -120,6 +120,7 @@ export interface CalculatedFieldDialogData { entityId: EntityId; debugLimitsConfiguration: string; tenantId: string; + entityName?: string; } export interface ArgumentEntityTypeParams { From d14d0d4e8a8f011df869fedd96c9a09964c60af4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 4 Feb 2025 17:03:25 +0200 Subject: [PATCH 130/438] used limits for state persistence --- .../server/actors/ActorSystemContext.java | 6 ++++++ .../CalculatedFieldEntityMessageProcessor.java | 4 ++-- .../CalculatedFieldManagerMessageProcessor.java | 6 +++--- .../service/cf/CalculatedFieldExecutionService.java | 2 +- .../service/cf/DefaultCalculatedFieldCache.java | 4 +++- .../cf/DefaultCalculatedFieldExecutionService.java | 4 ++-- .../service/cf/ctx/CalculatedFieldStateService.java | 3 ++- .../cf/ctx/state/BaseCalculatedFieldState.java | 5 +++-- .../service/cf/ctx/state/CalculatedFieldCtx.java | 10 +++++++++- .../service/cf/ctx/state/RocksDBStateService.java | 7 +++++-- .../ctx/state/ScriptCalculatedFieldStateTest.java | 12 ++++++++++-- .../ctx/state/SimpleCalculatedFieldStateTest.java | 13 ++++++++++++- .../common/data/cf/configuration/Argument.java | 6 ++++-- .../server/common/data/cf/configuration/Output.java | 2 ++ .../data/cf/configuration/ReferencedEntityKey.java | 4 ++-- 15 files changed, 66 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..49e2c9c46e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -98,6 +98,7 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 9392e3c02a..f6bc5b2d63 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -200,7 +200,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - if (state.isReady()) { + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { @@ -209,7 +209,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index ec86e640d4..31c8eee23c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -95,7 +95,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onFieldInitMsg(CalculatedFieldInitMsg msg) { var cf = msg.getCf(); - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -220,7 +220,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService()); + var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { @@ -248,7 +248,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { - var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService()); + var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getId()); List newCfList = new ArrayList<>(oldCfList.size()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 19f60165cf..393fbd3ec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,7 +43,7 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..c8f9a7e882 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -34,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -57,6 +58,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final AssetService assetService; private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ApiLimitService apiLimitService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); @@ -116,7 +118,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { if (ctx == null) { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { - ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); + ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..03c8f3de0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -265,8 +265,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void pushStateToStorage(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(stateId, state, callback); + public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + stateService.persistState(ctx, stateId, state, callback); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index c670e97580..e822d52767 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.cf.ctx; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Map; @@ -24,7 +25,7 @@ public interface CalculatedFieldStateService { Map restoreStates(); - void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); + void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 21105a44cb..86d83a1f70 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import lombok.AllArgsConstructor; import lombok.Data; import java.util.ArrayList; @@ -23,6 +24,7 @@ import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public abstract class BaseCalculatedFieldState implements CalculatedFieldState { protected List requiredArguments; @@ -34,8 +36,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { } public BaseCalculatedFieldState() { - this.requiredArguments = new ArrayList<>(); - this.arguments = new HashMap<>(); + this(new ArrayList<>(), new HashMap<>()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index 4600628838..c483c6ab5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -33,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; @@ -67,7 +69,10 @@ public class CalculatedFieldCtx { private boolean initialized; - public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService) { + private long maxDataPointsPerRollingArg; + private long maxStateSizeInKBytes; + + public CalculatedFieldCtx(CalculatedField calculatedField, TbelInvokeService tbelInvokeService, ApiLimitService apiLimitService) { this.calculatedField = calculatedField; this.cfId = calculatedField.getId(); @@ -96,6 +101,9 @@ public class CalculatedFieldCtx { this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; + + this.maxDataPointsPerRollingArg = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxDataPointsPerRollingArg); + this.maxStateSizeInKBytes = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getMaxStateSizeInKBytes); } public void init() { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 18e604118d..371751cc6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -59,8 +59,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { } @Override - public void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateProto stateProto = toProto(stateId, state); + if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + rocksDBService.put(toProto(stateId), toProto(stateId, state)); + } callback.onSuccess(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 42ff828dbd..bded0d058d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.thingsboard.script.api.tbel.DefaultTbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.AttributeScope; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -45,6 +47,8 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; @SpringBootTest(classes = DefaultTbelInvokeService.class) public class ScriptCalculatedFieldStateTest { @@ -64,9 +68,13 @@ public class ScriptCalculatedFieldStateTest { @Autowired private TbelInvokeService tbelInvokeService; + @MockBean + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), tbelInvokeService, apiLimitService); ctx.init(); state = new ScriptCalculatedFieldState(ctx.getArgNames()); } @@ -219,7 +227,7 @@ public class ScriptCalculatedFieldStateTest { ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("temperature", ArgumentType.TS_ROLLING, null); argument1.setRefEntityKey(refEntityKey1); argument1.setLimit(5); - argument1.setTimeWindow(30000); + argument1.setTimeWindow(30000L); Argument argument2 = new Argument(); ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("humidity", ArgumentType.TS_LATEST, null); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d6b384d85b..d803ad2dab 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -17,6 +17,9 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -32,6 +35,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import java.util.HashMap; @@ -41,7 +45,10 @@ import java.util.concurrent.ExecutionException; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; +@ExtendWith(MockitoExtension.class) public class SimpleCalculatedFieldStateTest { private final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("5b18e321-3327-4290-b996-d72a65e90382")); @@ -55,9 +62,13 @@ public class SimpleCalculatedFieldStateTest { private SimpleCalculatedFieldState state; private CalculatedFieldCtx ctx; + @Mock + private ApiLimitService apiLimitService; + @BeforeEach void setUp() { - ctx = new CalculatedFieldCtx(getCalculatedField(), null); + when(apiLimitService.getLimit(any(), any())).thenReturn(1000L); + ctx = new CalculatedFieldCtx(getCalculatedField(), null, apiLimitService); ctx.init(); state = new SimpleCalculatedFieldState(ctx.getArgNames()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java index b61c3bc507..9923751db6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java @@ -15,11 +15,13 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.springframework.lang.Nullable; import org.thingsboard.server.common.data.id.EntityId; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Argument { @Nullable @@ -27,7 +29,7 @@ public class Argument { private ReferencedEntityKey refEntityKey; private String defaultValue; - private int limit; - private long timeWindow; + private Integer limit; + private Long timeWindow; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java index 12cf97338a..b57b19d3ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data +@JsonInclude(JsonInclude.Include.NON_NULL) public class Output { private String name; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java index b4bcc77a17..fd0bf3ceb7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -15,18 +15,18 @@ */ package org.thingsboard.server.common.data.cf.configuration; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.server.common.data.AttributeScope; @Data @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) public class ReferencedEntityKey { private String key; private ArgumentType type; private AttributeScope scope; - - } From 555acff6b4ced9405b2ef2cfcc37e90a6b4d8fa4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:12:03 +0200 Subject: [PATCH 131/438] fix --- .../panel/calculated-field-argument-panel.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index b47d2f073c..fe60162005 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -173,7 +173,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { switch (this.entityId.entityType) { case EntityType.ASSET_PROFILE: return { - deviceTypes: [this.entityName], + assetTypes: [this.entityName], type: AliasFilterType.assetType }; case EntityType.DEVICE_PROFILE: From d185b784274f9ed19302c6bce1e5976fb29bd3ff Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 17:57:05 +0200 Subject: [PATCH 132/438] Resolved PR comments --- ...lated-field-arguments-table.component.html | 19 ++++++++------ ...lated-field-arguments-table.component.scss | 9 ++++--- .../calculated-field-dialog.component.ts | 7 +++--- ...lculated-field-argument-panel.component.ts | 25 +++---------------- .../shared/models/calculated-field.models.ts | 21 ++++++++++++++++ 5 files changed, 44 insertions(+), 37 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 24d67d8654..b25809ac82 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -87,19 +87,24 @@ (click)="manageArgument($event, button, $index)" [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> - edit - @if (argumentsFormArray.dirty - && group.get('refEntityKey').get('type').value === ArgumentType.Rolling - && calculatedFieldType() === CalculatedFieldType.SIMPLE) { - - } + + edit +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 8f8b1d5d81..3a1f1fbe25 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -19,9 +19,10 @@ font-size: 14px; } } - .edit-hint { - .mat-mdc-form-field-error { - font-size: 16px; - } +} + +:host { + .field-action { + color: rgba(0, 0, 0, 0.54); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 155a384273..55fc299475 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -36,7 +36,6 @@ import { EntityType } from '@shared/models/entity-type.models'; import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; -import { merge } from 'rxjs'; @Component({ selector: 'tb-calculated-field-dialog', @@ -60,10 +59,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(this.configFormGroup.get('arguments').value ?? this.data.value.configuration.arguments)) + startWith(this.data.value?.configuration?.arguments ?? {}), + map(argumentsObj => Object.keys(argumentsObj)) ); readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index fe60162005..510bdd95f3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -25,7 +25,8 @@ import { ArgumentType, ArgumentTypeTranslations, CalculatedFieldArgumentValue, - CalculatedFieldType + CalculatedFieldType, + getCalculatedFieldCurrentEntityFilter } from '@shared/models/calculated-field.models'; import { debounceTime, distinctUntilChanged, filter } from 'rxjs/operators'; import { EntityType } from '@shared/models/entity-type.models'; @@ -110,7 +111,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); - this.currentEntityFilter = this.getCurrentEntityFilter(); + this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); this.updateEntityFilter(this.argument.refEntityId?.entityType, true); this.toggleByEntityKeyType(this.argument.refEntityKey?.type); this.setInitialEntityKeyType(); @@ -169,26 +170,6 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.cd.markForCheck(); } - private getCurrentEntityFilter(): EntityFilter { - switch (this.entityId.entityType) { - case EntityType.ASSET_PROFILE: - return { - assetTypes: [this.entityName], - type: AliasFilterType.assetType - }; - case EntityType.DEVICE_PROFILE: - return { - deviceTypes: [this.entityName], - type: AliasFilterType.deviceType - }; - default: - return { - type: AliasFilterType.singleEntity, - singleEntity: this.entityId, - }; - } - } - private observeEntityFilterChanges(): void { merge( this.refEntityIdFormGroup.get('entityType').valueChanges, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index f73926f62b..fac1e9f942 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -20,6 +20,7 @@ import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; +import { AliasFilterType } from '@shared/models/alias.models'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { debugSettings?: EntityDebugSettings; @@ -133,3 +134,23 @@ export const ArgumentEntityTypeParamsMap =new Map { + switch (entityId.entityType) { + case EntityType.ASSET_PROFILE: + return { + assetTypes: [entityName], + type: AliasFilterType.assetType + }; + case EntityType.DEVICE_PROFILE: + return { + deviceTypes: [entityName], + type: AliasFilterType.deviceType + }; + default: + return { + type: AliasFilterType.singleEntity, + singleEntity: entityId, + }; + } +} From 33825e5660e5aeb39271904691a503d5c2232fd2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:03 +0200 Subject: [PATCH 133/438] Changed styling --- ...calculated-field-arguments-table.component.html | 7 +++---- ...calculated-field-arguments-table.component.scss | 14 ++++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index b25809ac82..4b7e516db0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 3a1f1fbe25..188371b0d8 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -13,6 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +:host { + .tb-form-table-row-cell-buttons{ + --mat-badge-legacy-small-size-container-size: 8px; + --mat-badge-small-size-container-overlap-offset: -5px; + --mat-badge-small-size-text-size: 0; + } +} + :host ::ng-deep { .tb-inline-field { a { @@ -20,9 +28,3 @@ } } } - -:host { - .field-action { - color: rgba(0, 0, 0, 0.54); - } -} From 1ac9d4137773e539880b6ba51063d8c5ddeac499 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 4 Feb 2025 18:24:34 +0200 Subject: [PATCH 134/438] formatting --- .../calculated-field-arguments-table.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index 188371b0d8..a0c90bba32 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .tb-form-table-row-cell-buttons{ + .tb-form-table-row-cell-buttons { --mat-badge-legacy-small-size-container-size: 8px; --mat-badge-small-size-container-overlap-offset: -5px; --mat-badge-small-size-text-size: 0; From b2bf5b9f6535bc55c233d4ba7ba13edb156bc08b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 12:20:23 +0200 Subject: [PATCH 135/438] configurable edqs sync batch sizes --- .../server/service/edqs/DefaultEdqsService.java | 2 +- .../server/service/edqs/EdqsListener.java | 2 +- .../server/service/edqs/EdqsSyncService.java | 16 +++++++++------- .../service/edqs/KafkaEdqsSyncService.java | 2 +- .../service/edqs/LocalEdqsSyncService.java | 2 +- application/src/main/resources/thingsboard.yml | 5 ++++- .../EdqsEntityQueryControllerTest.java | 2 +- .../service/entitiy/EdqsEntityServiceTest.java | 2 +- .../test/resources/application-test.properties | 2 +- .../server/queue/edqs/EdqsComponent.java | 2 +- .../server/queue/edqs/InMemoryEdqsComponent.java | 2 +- .../server/queue/edqs/KafkaEdqsComponent.java | 2 +- .../server/dao/sql/query/DummyEdqsService.java | 2 +- 13 files changed, 24 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java index 160d4bb565..e7243ed3bc 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/DefaultEdqsService.java @@ -81,7 +81,7 @@ import java.util.concurrent.TimeUnit; @Service @RequiredArgsConstructor @Slf4j -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class DefaultEdqsService implements EdqsService { private final EdqsClientQueueFactory queueFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java index c4ce0c7a7e..dc8d4bf36d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsListener.java @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; @Service @RequiredArgsConstructor -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "true") +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "true") public class EdqsListener { private final EdqsService edqsService; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index c2750ae840..80aeee8635 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.edqs; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; @@ -42,7 +43,6 @@ import org.thingsboard.server.dao.model.sqlts.dictionary.KeyDictionaryEntry; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.tenant.TenantDao; import java.util.List; import java.util.Map; @@ -58,11 +58,13 @@ import static org.thingsboard.server.common.data.ObjectType.edqsTenantTypes; @Slf4j public abstract class EdqsSyncService { + @Value("${queue.edqs.sync.entity_batch_size:10000}") + private int entityBatchSize; + @Value("${queue.edqs.sync.ts_batch_size:10000}") + private int tsBatchSize; @Autowired private EntityDaoRegistry entityDaoRegistry; @Autowired - private TenantDao tenantDao; - @Autowired private AttributesDao attributesDao; @Autowired private KeyDictionaryDao keyDictionaryDao; @@ -112,7 +114,7 @@ public abstract class EdqsSyncService { Dao dao = entityDaoRegistry.getDao(entityType); UUID lastId = UUID.fromString("00000000-0000-0000-0000-000000000000"); while (true) { - var batch = dao.findNextBatch(lastId, 10000); + var batch = dao.findNextBatch(lastId, entityBatchSize); if (batch.isEmpty()) { break; } @@ -140,7 +142,7 @@ public abstract class EdqsSyncService { while (true) { List batch = relationRepository.findNextBatch(lastFromEntityId, lastFromEntityType, lastRelationTypeGroup, - lastRelationType, lastToEntityId, lastToEntityType, 10000); + lastRelationType, lastToEntityId, lastToEntityType, entityBatchSize); if (batch.isEmpty()) { break; } @@ -189,7 +191,7 @@ public abstract class EdqsSyncService { int lastAttributeKey = Integer.MIN_VALUE; while (true) { - List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, 10000); + List batch = attributesDao.findNextBatch(lastEntityId, lastAttributeType, lastAttributeKey, tsBatchSize); if (batch.isEmpty()) { break; } @@ -228,7 +230,7 @@ public abstract class EdqsSyncService { int lastKey = Integer.MIN_VALUE; while (true) { - List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, 10000); + List batch = tsKvLatestRepository.findNextBatch(lastEntityId, lastKey, tsBatchSize); if (batch.isEmpty()) { break; } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java index f4e9a02b45..0c6f948770 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/KafkaEdqsSyncService.java @@ -27,7 +27,7 @@ import java.util.Collections; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'kafka'") public class KafkaEdqsSyncService extends EdqsSyncService { private final TbKafkaSettings kafkaSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java index 924bf94830..11d5894307 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/LocalEdqsSyncService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.edqs.util.EdqsRocksDb; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}' == 'true' && '${queue.type:null}' == 'in-memory'") public class LocalEdqsSyncService extends EdqsSyncService { private final EdqsRocksDb db; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index dd0e5948c7..e41d0c2aed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1695,7 +1695,10 @@ queue: # Statistics printing interval for Housekeeper print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" edqs: - sync_enabled: "${TB_EDQS_SYNC_ENABLED:true}" # FIXME: disable by default before release + sync: + enabled: "${TB_EDQS_SYNC_ENABLED:true}" # Enable/disable EDQS synchronization with postgres db FIXME: disable by default before release + entity_batch_size: "${TB_EDQS_SYNC_ENTITY_BATCH_SIZE:10000}" # batch size of entities being synced with EDQS + ts_batch_size: "${TB_EDQS_SYNC_TS_BATCH_SIZE:10000}" # batch size of timeseries data being synced with EDQS api_enabled: "${TB_EDQS_API_ENABLED:true}" # FIXME: disable by default before release mode: "${TB_EDQS_MODE:local}" # local or remote local: diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index baf9f98993..9bb0bbe30a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -36,7 +36,7 @@ import static org.awaitility.Awaitility.await; @TestPropertySource(properties = { // "queue.type=kafka", // uncomment to use Kafka // "queue.kafka.bootstrap.servers=10.7.1.254:9092", - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index cf24deb7e4..cb377e0431 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -34,7 +34,7 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest @TestPropertySource(properties = { - "queue.edqs.sync_enabled=true", + "queue.edqs.sync.enabled=true", "queue.edqs.api_enabled=true", "queue.edqs.mode=local" }) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index ba6863c705..8933c36db5 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -57,5 +57,5 @@ server.log_controller_error_stack_trace=true transport.gateway.dashboard.sync.enabled=false -queue.edqs.sync_enabled=false +queue.edqs.sync.enabled=false queue.edqs.api_enabled=false diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java index 838f9b4aa2..27a1e09f79 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsComponent.java @@ -22,7 +22,7 @@ import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) // TODO: tb-core ? -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local'))") public @interface EdqsComponent { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java index 5055787fde..3e04dda3b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/InMemoryEdqsComponent.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && '${service.type:null}'=='monolith' && '${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='in-memory'") public @interface InMemoryEdqsComponent { } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java index 9f112e7f59..9963972e9c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/KafkaEdqsComponent.java @@ -21,7 +21,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${queue.edqs.sync_enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + +@ConditionalOnExpression("'${queue.edqs.sync.enabled:true}'=='true' && ('${service.type:null}'=='edqs' || " + "(('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core') && " + "'${queue.edqs.mode:null}'=='local' && '${queue.type:null}'=='kafka'))") public @interface KafkaEdqsComponent { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index 91934293a6..b69aa52ec5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.edqs.EdqsService; @Service -@ConditionalOnProperty(value = "queue.edqs.sync_enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(value = "queue.edqs.sync.enabled", havingValue = "false", matchIfMissing = true) public class DummyEdqsService implements EdqsService { @Override From 3818d1cb68d35129e535d96f2ddb098c59812bdb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Feb 2025 12:34:10 +0200 Subject: [PATCH 136/438] debug events(wip) --- .../server/actors/ActorSystemContext.java | 95 ++++++++++++------- .../cf/ctx/state/RocksDBStateService.java | 3 +- .../permission/TenantAdminPermissions.java | 1 + .../src/main/resources/thingsboard.yml | 6 ++ .../dao/cf/BaseCalculatedFieldService.java | 2 +- .../CalculatedFieldDebugEventRepository.java | 2 +- .../main/resources/sql/schema-entities.sql | 4 +- 7 files changed, 72 insertions(+), 41 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 49e2c9c46e..652cdabf77 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -137,6 +137,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Slf4j @Component @@ -179,6 +180,8 @@ public class ActorSystemContext { private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); + private final ConcurrentMap cfDebugPerTenantLimits = new ConcurrentHashMap<>(); + public ConcurrentMap getDebugPerTenantLimits() { return debugPerTenantLimits; } @@ -441,6 +444,11 @@ public class ActorSystemContext { @Getter private TbCoreToTransportService tbCoreToTransportService; + @Lazy + @Autowired(required = false) + @Getter + private ApiLimitService apiLimitService; + /** * The following Service will be null if we operate in tb-core mode */ @@ -517,11 +525,6 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; - @Lazy - @Autowired(required = false) - @Getter - private ApiLimitService apiLimitService; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; @@ -625,6 +628,14 @@ public class ActorSystemContext { @Getter private String deviceStateNodeRateLimitConfig; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean cfDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String cfDebugPerTenantLimitsConfiguration; + @Getter @Setter private TbActorSystem actorSystem; @@ -753,37 +764,6 @@ public class ActorSystemContext { } } - public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(entityId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString(arguments)); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); - } - - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); - } - } - private boolean checkLimits(TenantId tenantId, TbMsg tbMsg, Throwable error) { if (debugPerTenantEnabled) { DebugTbRateLimits debugTbRateLimits = debugPerTenantLimits.computeIfAbsent(tenantId, id -> @@ -817,6 +797,49 @@ public class ActorSystemContext { Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } + public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { + if (cfDebugPerTenantEnabled) { + TbRateLimits rateLimits = cfDebugPerTenantLimits.computeIfAbsent(tenantId, id -> new TbRateLimits(cfDebugPerTenantLimitsConfiguration)); + + if (rateLimits.tryConsume()) { + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getValue())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(toString(error)); + } + + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } + if (log.isTraceEnabled()) { + log.trace("[{}] Tenant level debug mode rate limit detected: {}", tenantId, calculatedFieldId); + } + } + } + } + public static Exception toException(Throwable error) { return Exception.class.isInstance(error) ? (Exception) error : new Exception(error); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 371751cc6f..8a6a5c9cb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -61,7 +61,8 @@ public class RocksDBStateService implements CalculatedFieldStateService { @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); - if (stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { rocksDBService.put(toProto(stateId), toProto(stateId, state)); } callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 5b98a56f24..b0e35e57e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -55,6 +55,7 @@ public class TenantAdminPermissions extends AbstractPermissions { put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, new PermissionChecker.GenericPermissionChecker(Operation.READ)); put(Resource.MOBILE_APP, tenantEntityPermissionChecker); put(Resource.MOBILE_APP_BUNDLE, tenantEntityPermissionChecker); + put(Resource.CALCULATED_FIELD, tenantEntityPermissionChecker); } public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..197f05a5d5 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CF_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index d37a4a53c1..b062a2de5e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -62,9 +62,9 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements try { TenantId tenantId = calculatedField.getTenantId(); log.trace("Executing save calculated field, [{}]", calculatedField); + updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); CalculatedField savedCalculatedField = calculatedFieldDao.save(tenantId, calculatedField); createOrUpdateCalculatedFieldLink(tenantId, savedCalculatedField); - updateDebugSettings(tenantId, calculatedField, System.currentTimeMillis()); eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(savedCalculatedField.getTenantId()).entityId(savedCalculatedField.getId()) .entity(savedCalculatedField).oldEntity(oldCalculatedField).created(calculatedField.getId() == null).build()); return savedCalculatedField; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java index c0bd21ca74..a759babcfb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/CalculatedFieldDebugEventRepository.java @@ -35,7 +35,7 @@ public interface CalculatedFieldDebugEventRepository extends EventRepository findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override - @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + @Query("SELECT e FROM CalculatedFieldDebugEventEntity e WHERE " + "e.tenantId = :tenantId " + "AND e.entityId = :entityId " + "AND (:startTime IS NULL OR e.ts >= :startTime) " + diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a6d1e8800d..4b395a5768 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -955,10 +955,10 @@ CREATE TABLE IF NOT EXISTS cf_debug_event ( id uuid NOT NULL, tenant_id uuid NOT NULL , ts bigint NOT NULL, - entity_id uuid NOT NULL, + entity_id uuid NOT NULL, -- calculated field id service_id varchar, cf_id uuid NOT NULL, - e_entity_id uuid, + e_entity_id uuid, -- target entity id e_entity_type varchar, e_msg_id uuid, e_msg_type varchar, From 407511be52b1f7de51a74a62cf6d58d7c4ff12aa Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 5 Feb 2025 12:51:50 +0200 Subject: [PATCH 137/438] Cache implementation --- .../server/actors/ActorSystemContext.java | 6 + ...alculatedFieldManagerMessageProcessor.java | 44 ++---- .../service/cf/CalculatedFieldCache.java | 4 - .../cf/DefaultCalculatedFieldCache.java | 60 ++------- ...efaultCalculatedFieldExecutionService.java | 125 ------------------ .../cf/DefaultCalculatedFieldInitService.java | 36 ++--- .../CalculatedFieldEntityProfileCache.java | 35 +++++ ...aultCalculatedFieldEntityProfileCache.java | 86 ++++++++++++ .../cf/cache/TenantEntityProfileCache.java | 118 +++++++++++++++++ .../server/dao/asset/AssetService.java | 3 + .../server/dao/device/DeviceService.java | 3 + .../common/data/ProfileEntityIdInfo.java | 55 ++++++++ .../server/queue/util/AfterStartUp.java | 7 +- .../server/dao/asset/AssetDao.java | 3 + .../server/dao/asset/BaseAssetService.java | 8 ++ .../server/dao/device/DeviceDao.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 8 ++ .../server/dao/sql/asset/JpaAssetDao.java | 12 ++ .../sql/device/AbstractNativeRepository.java | 54 ++++++++ .../device/DefaultNativeAssetRepository.java | 55 ++++++++ .../device/DefaultNativeDeviceRepository.java | 52 ++++---- .../server/dao/sql/device/JpaDeviceDao.java | 7 + .../dao/sql/device/NativeAssetRepository.java | 20 +++ .../sql/device/NativeDeviceRepository.java | 4 +- .../device/NativeProfileEntityRepository.java | 26 ++++ 25 files changed, 578 insertions(+), 257 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index daa6744fd9..f581d0b266 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,6 +106,7 @@ import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -516,6 +517,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldEntityProfileCache calculatedFieldEntityProfileCache; + @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private long maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index b2ff883236..cd2e5e876f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -26,14 +26,11 @@ import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; @@ -42,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -50,7 +48,6 @@ import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -69,19 +66,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map calculatedFields = new HashMap<>(); private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; protected TbActorCtx ctx; final TenantId tenantId; - private final static int initFetchPackSize = 1024; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); + this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldExecutionService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); @@ -173,8 +170,10 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); + EntityId profileId = getProfileId(tenantId, entityId); + cfEntityCache.add(tenantId, entityId, profileId); var entityIdFields = getCalculatedFieldsByEntityId(entityId); - var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); + var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); if (fieldsCount > 0) { MultipleTbCallback multiCallback = new MultipleTbCallback(fieldsCount, callback); @@ -187,6 +186,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { if (msg.getOldProfileId() != null && msg.getOldProfileId() != msg.getProfileId()) { + cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); var oldProfileCfs = getCalculatedFieldsByEntityId(msg.getOldProfileId()); var newProfileCfs = getCalculatedFieldsByEntityId(msg.getProfileId()); var fieldsCount = oldProfileCfs.size() + newProfileCfs.size(); @@ -202,6 +202,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + cfEntityCache.evict(tenantId, msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } @@ -288,7 +289,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = getEntitiesByProfile(entityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { //TODO: no need to do this if we cache all created actors and know which one belong to us; var multiCallback = new MultipleTbCallback(entityIds.size(), callback); @@ -335,7 +336,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var cf = calculatedFields.get(link.cfId()); if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { // iterate over all entities that belong to profile and push the message for corresponding CF - var entityIds = getEntitiesByProfile(targetEntityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); if (!entityIds.isEmpty()) { MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); @@ -392,34 +393,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return result; } - private Set getEntitiesByProfile(EntityId entityProfileId) { - Set entities = profileEntities.get(entityProfileId); - if (entities == null) { - entities = switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - systemContext.getAssetService().findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - systemContext.getDeviceService().findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); - return entities; - } - private void initCf(CalculatedFieldCtx cfCtx, TbCallback callback, boolean forceStateReinit) { EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = getEntitiesByProfile(entityId); + var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { var multiCallback = new MultipleTbCallback(entityIds.size(), callback); entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ff3bda5da5..96a3694543 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -31,16 +31,12 @@ public interface CalculatedFieldCache { List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(EntityId entityId); CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId); List getCalculatedFieldCtxsByEntityId(EntityId entityId); - Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); - void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 7e841a0cf8..ed5ae23a17 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -15,31 +15,28 @@ */ package org.thingsboard.server.service.cf; -import jakarta.annotation.PostConstruct; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -51,32 +48,32 @@ import java.util.concurrent.locks.ReentrantLock; @RequiredArgsConstructor public class DefaultCalculatedFieldCache implements CalculatedFieldCache { + private static final Integer UNKNOWN_PARTITION = -1; + private final Lock calculatedFieldFetchLock = new ReentrantLock(); private final CalculatedFieldService calculatedFieldService; - private final AssetService assetService; - private final DeviceService deviceService; private final TbelInvokeService tbelInvokeService; + private final ActorSystemContext actorSystemContext; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); - private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; - @PostConstruct + @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); + cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() @@ -84,6 +81,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); + cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); } @Override @@ -96,11 +94,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } - @Override - public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); - } - @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); @@ -139,39 +132,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { .toList(); } - @Override - public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { - Set entities = profileEntities.get(entityProfileId); - if (entities == null) { - calculatedFieldFetchLock.lock(); - try { - entities = profileEntities.get(entityProfileId); - if (entities == null) { - entities = switch (entityProfileId.getEntityType()) { - case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set assetIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add); - return assetIds; - }); - case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> { - Set deviceIds = new HashSet<>(); - (new PageDataIterable<>(pageLink -> - deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add); - return deviceIds; - }); - default -> - throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE."); - }; - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities); - return entities; - } - @Override public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { calculatedFieldFetchLock.lock(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index d8e42c916d..5c4ab5ef6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -277,48 +277,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); -// PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); -// Map> tpiTargetEntityMap = new HashMap<>(); -// -// for (CalculatedField cf : cfs) { -// -// Consumer resolvePartition = entityId -> { -// TopicPartitionInfo tpi; -// try { -// tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); -// if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { -// tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); -// } -// } catch (Exception e) { -// log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", -// entityId, cf.getTenantId(), e.getMessage()); -// } -// }; -// -// EntityId cfEntityId = cf.getEntityId(); -// if (isProfileEntity(cfEntityId)) { -// calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId).forEach(resolvePartition); -// } else { -// resolvePartition.accept(cfEntityId); -// } -// } -// -// for (var entry : tpiTargetEntityMap.entrySet()) { -// for (List partition : Lists.partition(entry.getValue(), 1000)) { -// log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size()); -// var future = calculatedFieldExecutor.submit(() -> { -// try { -// for (CalculatedFieldEntityCtxId ctxId : partition) { -// restoreState(ctxId.cfId(), ctxId.entityId()); -// } -// } catch (Throwable t) { -// log.error("Unexpected exception while restoring CalculatedField states", t); -// throw t; -// } -// }); -// result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future); -// } -// } return result; } @@ -331,89 +289,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } -// @Override -// public void onEntityUpdateMsg(CalculatedFieldEntityUpdateMsgProto proto, TbCallback callback) { -// try { -// TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); -// -// TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); -// if (tpi.isMyPartition()) { -// log.info("Received CalculatedFieldEntityUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); -// if (proto.getDeleted()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity deleted from profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); -// calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); -// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); -// } -// if (proto.getAdded()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity added to profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); -// initializeStateForEntityByProfile(entityId, newProfileId, callback); -// } -// if (proto.getUpdated()) { -// log.info("Executing CalculatedFieldEntityUpdateMsgProto msg: entity changed the profile, tenantId=[{}], entityId=[{}]", tenantId, entityId); -// -// EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); -// EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); -// -// calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); -// initializeStateForEntityByProfile(entityId, newProfileId, callback); -// } -// } else { -// clusterService.pushNotificationToCalculatedFields(tenantId, entityId, ToCalculatedFieldNotificationMsg.newBuilder().setEntityUpdateMsg(proto).build(), null); -// } -// } catch (Exception e) { -// log.trace("Failed to process entity update msg: [{}]", proto, e); -// } -// } - - private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - } - - private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() - .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId())) - .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); - } - - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { - initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); - } - - private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map commonArguments, TbCallback callback) { - Map argumentValues = new HashMap<>(commonArguments); - List> futures = new ArrayList<>(); - - calculatedFieldCtx.getArguments().forEach((key, argument) -> { - if (!commonArguments.containsKey(key)) { - futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - } - }); - - Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { - @Override - public void onSuccess(List results) { -// updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>()); - callback.onSuccess(); - } - - @Override - public void onFailure(Throwable t) { - log.error("Failed to initialize state for entity: [{}]", entityId, t); - callback.onFailure(t); - } - }, calculatedFieldCallbackExecutor); - } - - @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 87dda7013f..f71617e204 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -17,41 +17,49 @@ package org.thingsboard.server.service.cf; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; -import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +@Slf4j @Service @TbRuleEngineComponent @RequiredArgsConstructor public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { - private final CalculatedFieldService calculatedFieldService; + private final CalculatedFieldEntityProfileCache entityProfileCache; private final CalculatedFieldStateService stateService; + private final ActorSystemContext actorSystemContext; + private final AssetService assetService; + private final DeviceService deviceService; @Value("${calculated_fields.init_fetch_pack_size:50000}") @Getter private int initFetchPackSize; - @AfterStartUp(order = AfterStartUp.CF_INIT_SERVICE) + @AfterStartUp(order = AfterStartUp.CF_READ_PROFILE_ENTITIES_SERVICE) public void initCalculatedFieldDefinitions() { - PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); - PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); - //TODO: combine with the DefaultCalculatedFieldCache. - + PageDataIterable deviceIdInfos = new PageDataIterable<>(deviceService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : deviceIdInfos) { + log.trace("Processing device record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } + PageDataIterable assetIdInfos = new PageDataIterable<>(assetService::findProfileEntityIdInfos, initFetchPackSize); + for (ProfileEntityIdInfo idInfo : assetIdInfos) { + log.trace("Processing asset record: {}", idInfo); + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } } @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) @@ -59,6 +67,4 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); } - - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..6b0718eb84 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +import java.util.Collection; + +public interface CalculatedFieldEntityProfileCache extends ApplicationListener { + + void add(TenantId tenantId, EntityId profileId, EntityId entityId); + + void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); + + void evict(TenantId tenantId, EntityId entityId); + + Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java new file mode 100644 index 0000000000..13f95c547d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.discovery.HashPartitionService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +@TbRuleEngineComponent +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { + + private static final Integer UNKNOWN = -1; + private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); + private final PartitionService partitionService; + private volatile List myPartitions = Collections.emptyList(); + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + myPartitions = event.getCalculatedFieldsPartitions().stream() + .filter(TopicPartitionInfo::isMyPartition) + .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); + //Naive approach that need to be improved. + tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); + } + + @Override + public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) + .add(profileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var partition = tpi.getPartition().orElse(UNKNOWN); + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + //TODO: make this method atomic; + cache.remove(oldProfileId, entityId); + cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); + } + + @Override + public void evict(TenantId tenantId, EntityId entityId) { + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + cache.removeEntityId(entityId); + } + + @Override + public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { + return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java new file mode 100644 index 0000000000..b3499f3cc4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.cache; + +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TenantEntityProfileCache { + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Map>> allEntities = new HashMap<>(); + private final Map> myEntities = new HashMap<>(); + + public void setMyPartitions(List myPartitions) { + lock.writeLock().lock(); + try { + myEntities.clear(); + myPartitions.forEach(partitionId -> { + var map = allEntities.get(partitionId); + if (map != null) { + map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); + } + }); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeProfileId(EntityId profileId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.remove(profileId)); + // Remove from myEntities + myEntities.remove(profileId); + } finally { + lock.writeLock().unlock(); + } + } + + public void removeEntityId(EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); + // Remove from myEntities + myEntities.values().forEach(set -> set.remove(entityId)); + } finally { + lock.writeLock().unlock(); + } + } + + public void remove(EntityId profileId, EntityId entityId) { + lock.writeLock().lock(); + try { + // Remove from allEntities + allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); + // Remove from myEntities + removeSafely(myEntities, profileId, entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { + lock.writeLock().lock(); + try { + if (mine) { + myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); + } + allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); + } finally { + lock.writeLock().unlock(); + } + } + + public Collection getMyEntityIdsByProfileId(EntityId profileId) { + lock.readLock().lock(); + try { + var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); + List result = new ArrayList<>(entities.size()); + result.addAll(entities); + return result; + } finally { + lock.readLock().unlock(); + } + } + + private void removeSafely(Map> map, EntityId profileId, EntityId entityId) { + var set = map.get(profileId); + if (set != null) { + set.remove(entityId); + } + } +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 2a9fe08827..15ed81d4c2 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.asset.AssetSearchQuery; @@ -63,6 +64,8 @@ public interface AssetService extends EntityDaoService { PageData findAssetInfosByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 0848e5a1cb..eb018aef1d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -73,6 +74,8 @@ public interface DeviceService extends EntityDaoService { PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java new file mode 100644 index 0000000000..83b56c38a7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ProfileEntityIdInfo.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@Slf4j +public class ProfileEntityIdInfo implements Serializable, HasTenantId { + + private static final long serialVersionUID = 8532058281983868003L; + + private final TenantId tenantId; + private final EntityId profileId; + private final EntityId entityId; + + private ProfileEntityIdInfo(UUID tenantId, EntityId profileId, EntityId entityId) { + this.tenantId = TenantId.fromUUID(tenantId); + this.profileId = profileId; + this.entityId = entityId; + } + + public static ProfileEntityIdInfo create(UUID tenantId, DeviceProfileId profileId, DeviceId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + + public static ProfileEntityIdInfo create(UUID tenantId, AssetProfileId profileId, AssetId entityId) { + return new ProfileEntityIdInfo(tenantId, profileId, entityId); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 70ae853e1e..f97c3b0a1b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -38,9 +38,10 @@ public @interface AfterStartUp { int ACTOR_SYSTEM = 9; int REGULAR_SERVICE = 10; - int CF_INIT_SERVICE = 10; - int CF_STATE_RESTORE_SERVICE = 11; - int CF_CONSUMER_SERVICE = 12; + int CF_READ_PROFILE_ENTITIES_SERVICE = 10; + int CF_READ_CF_SERVICE = 11; + int CF_STATE_RESTORE_SERVICE = 12; + int CF_CONSUMER_SERVICE = 13; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 8e40ec6d87..783a084ef3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.asset; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -236,4 +237,6 @@ public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityD PageData findAssetsByTenantIdAndEdgeIdAndType(UUID tenantId, UUID edgeId, String type, PageLink pageLink); PageData> getAllAssetTypes(PageLink pageLink); + + PageData findProfileEntityIdInfos(PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 564da127ab..607d09e23c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -26,6 +26,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; @@ -285,6 +286,13 @@ public class BaseAssetService extends AbstractCachedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return assetDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 7c82df7703..b474e0b8e5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -230,5 +231,8 @@ public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntit PageData findDeviceIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ffebd5f032..99ae252f79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.audit.ActionType; @@ -386,6 +387,13 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfos(PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfos, pageLink [{}]", pageLink); + validatePageLink(pageLink); + return deviceDao.findProfileEntityIdInfos(pageLink); + } + @Override public PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndType, tenantId [{}], type [{}], pageLink [{}]", tenantId, type, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index db6472f825..650d991521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.AssetId; @@ -35,6 +36,8 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; +import org.thingsboard.server.dao.sql.device.NativeAssetRepository; +import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; @@ -56,6 +59,9 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Autowired private AssetRepository assetRepository; + @Autowired + private NativeAssetRepository nativeAssetRepository; + @Autowired private AssetProfileRepository assetProfileRepository; @@ -252,6 +258,12 @@ public class JpaAssetDao extends JpaAbstractDao implements A DaoUtil.toPageable(pageLink, Arrays.asList(new SortOrder("tenantId"), new SortOrder("type"))))); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeAssetRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java new file mode 100644 index 0000000000..bde9278cd4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/AbstractNativeRepository.java @@ -0,0 +1,54 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@RequiredArgsConstructor +@Repository +@Slf4j +public class AbstractNativeRepository { + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final TransactionTemplate transactionTemplate; + + protected PageData find(String countQuery, String findQuery, Pageable pageable, Function, T> mapper) { + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + int totalElements = jdbcTemplate.queryForObject(countQuery, Collections.emptyMap(), Integer.class); + log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); + startTs = System.currentTimeMillis(); + List> rows = jdbcTemplate.queryForList(String.format(findQuery, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); + log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); + int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; + boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); + var data = rows.stream().map(mapper).collect(Collectors.toList()); + return new PageData<>(data, totalPages, totalElements, hasNext); + }); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java new file mode 100644 index 0000000000..43f80e0a86 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.domain.Pageable; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.UUID; + +@Repository +@Slf4j +public class DefaultNativeAssetRepository extends AbstractNativeRepository implements NativeAssetRepository { + + private final String COUNT_QUERY = "SELECT count(id) FROM asset;"; + + public DefaultNativeAssetRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + AssetId id = new AssetId((UUID) row.get("id")); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 002c796089..67b45ba70d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -15,50 +15,50 @@ */ package org.thingsboard.server.dao.sql.device; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; -@RequiredArgsConstructor @Repository @Slf4j -public class DefaultNativeDeviceRepository implements NativeDeviceRepository { +public class DefaultNativeDeviceRepository extends AbstractNativeRepository implements NativeDeviceRepository { private final String COUNT_QUERY = "SELECT count(id) FROM device;"; - private final String QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; - private final NamedParameterJdbcTemplate jdbcTemplate; - private final TransactionTemplate transactionTemplate; + + public DefaultNativeDeviceRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) { + super(jdbcTemplate, transactionTemplate); + } @Override public PageData findDeviceIdInfos(Pageable pageable) { - return transactionTemplate.execute(status -> { - long startTs = System.currentTimeMillis(); - int totalElements = jdbcTemplate.queryForObject(COUNT_QUERY, Collections.emptyMap(), Integer.class); - log.debug("Count query took {} ms", System.currentTimeMillis() - startTs); - startTs = System.currentTimeMillis(); - List> rows = jdbcTemplate.queryForList(String.format(QUERY, pageable.getPageSize(), pageable.getOffset()), Collections.emptyMap()); - log.debug("Main query took {} ms", System.currentTimeMillis() - startTs); - int totalPages = pageable.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageable.getPageSize()) : 1; - boolean hasNext = pageable.getPageSize() > 0 && totalElements > pageable.getOffset() + rows.size(); - var data = rows.stream().map(row -> { - UUID id = (UUID) row.get("id"); - var tenantIdObj = row.get("tenantId"); - var customerIdObj = row.get("customerId"); - return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); - }).collect(Collectors.toList()); - return new PageData<>(data, totalPages, totalElements, hasNext); + String DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, customer_id as customerId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, DEVICE_ID_INFO_QUERY, pageable, row -> { + UUID id = (UUID) row.get("id"); + var tenantIdObj = row.get("tenantId"); + var customerIdObj = row.get("customerId"); + return new DeviceIdInfo(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), customerIdObj != null ? (UUID) customerIdObj : null, id); }); } + + @Override + public PageData findProfileEntityIdInfos(Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + DeviceId id = new DeviceId((UUID) row.get("id")); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index bf0def9fca..74afdca880 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -272,6 +273,12 @@ public class JpaDeviceDao extends JpaAbstractDao implement return nativeDeviceRepository.findDeviceIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfos(PageLink pageLink) { + log.debug("Find profile device id infos by pageLink [{}]", pageLink); + return nativeDeviceRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); + } + @Override public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java new file mode 100644 index 0000000000..6a0539ee9d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeAssetRepository.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +public interface NativeAssetRepository extends NativeProfileEntityRepository { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java index f968749461..925bf12618 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeDeviceRepository.java @@ -17,10 +17,12 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.domain.Pageable; import org.thingsboard.server.common.data.DeviceIdInfo; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageData; -public interface NativeDeviceRepository { +public interface NativeDeviceRepository extends NativeProfileEntityRepository { PageData findDeviceIdInfos(Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java new file mode 100644 index 0000000000..1d3dbbec45 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.device; + +import org.springframework.data.domain.Pageable; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; +import org.thingsboard.server.common.data.page.PageData; + +public interface NativeProfileEntityRepository { + + PageData findProfileEntityIdInfos(Pageable pageable); + +} From 3d9897f5af21ae26f197009cd6e985e5316f10ee Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 13:30:42 +0200 Subject: [PATCH 138/438] fixed NullPointer exception in CachedAttributesService --- .../server/dao/attributes/CachedAttributesService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index c5ac50c9f1..3c12ae3209 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -266,7 +266,9 @@ public class CachedAttributesService implements AttributesService { String key = keyVersionPair.getFirst(); Long version = keyVersionPair.getSecond(); cache.evict(new AttributeCacheKey(scope, entityId, key), version); - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } return key; }, cacheExecutor)).collect(Collectors.toList())); } From bf9d4d30ae7313a6f150f5a20a6cd484af0555fb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 5 Feb 2025 14:21:41 +0200 Subject: [PATCH 139/438] handled debug error event --- .../CalculatedFieldEntityMessageProcessor.java | 14 ++++++++++---- .../org/thingsboard/common/util/DebugModeUtil.java | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index f6bc5b2d63..309dfde2e9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -201,10 +201,16 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { if (state.isReady() && ctx.isInitialized()) { - CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); - if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + try { + CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } else { callback.onSuccess(); // State was updated but no calculation performed; diff --git a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java index 9ba9038594..d63f846dd2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/DebugModeUtil.java @@ -57,4 +57,14 @@ public final class DebugModeUtil { return debugSettings != null && nodeConnections != null && debugSettings.isFailuresEnabled() && nodeConnections.contains(TbNodeConnectionType.FAILURE); } } + + public static boolean isDebugFailuresAvailable(HasDebugSettings debugSettingsAware) { + if (isDebugAllAvailable(debugSettingsAware)) { + return true; + } else { + var debugSettings = debugSettingsAware.getDebugSettings(); + return debugSettings != null && debugSettings.isFailuresEnabled(); + } + } + } From 0332e0f0c953d4e5d7b37d46207177e119631a0d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 5 Feb 2025 16:39:48 +0200 Subject: [PATCH 140/438] Bug fixes for monolith --- .../server/actors/ActorSystemContext.java | 1 + .../CalculatedFieldEntityActorCreator.java | 3 +- ...alculatedFieldManagerMessageProcessor.java | 35 +++++++++++++++---- .../cf/DefaultCalculatedFieldCache.java | 19 ++++++---- ...efaultCalculatedFieldExecutionService.java | 1 - .../cf/cache/TenantEntityProfileCache.java | 4 +++ .../cf/ctx/state/CalculatedFieldCtx.java | 12 ++++--- .../entitiy/EntityStateSourcingListener.java | 8 ++--- .../queue/DefaultTbClusterService.java | 13 ++++--- .../server/actors/TbActorMailbox.java | 3 +- 10 files changed, 70 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 978b2499a4..8f15a6766b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -314,6 +314,7 @@ public class ActorSystemContext { @Getter private TbEntityViewService tbEntityViewService; + @Lazy @Autowired @Getter private TelemetrySubscriptionService tsSubService; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java index c5f8ecf046..1dfe92c4bc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActorCreator.java @@ -18,6 +18,7 @@ package org.thingsboard.server.actors.calculatedField; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActor; import org.thingsboard.server.actors.TbActorId; +import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.TbEntityActorId; import org.thingsboard.server.actors.device.DeviceActor; import org.thingsboard.server.actors.service.ContextBasedCreator; @@ -38,7 +39,7 @@ public class CalculatedFieldEntityActorCreator extends ContextBasedCreator { @Override public TbActorId createActorId() { - return new TbEntityActorId(entityId); + return new TbCalculatedFieldEntityActorId(entityId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 34608337a5..c219544a91 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -24,6 +24,7 @@ import org.thingsboard.server.actors.TbCalculatedFieldEntityActorId; import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -50,7 +51,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -175,7 +175,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); EntityId profileId = getProfileId(tenantId, entityId); - cfEntityCache.add(tenantId, entityId, profileId); + cfEntityCache.add(tenantId, profileId, entityId); var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(profileId); var fieldsCount = entityIdFields.size() + profileIdFields.size(); @@ -233,6 +233,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cfCtx); + addLinks(cf); initCf(cfCtx, callback, false); } } @@ -251,7 +252,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); calculatedFields.put(newCf.getId(), newCfCtx); - List oldCfList = entityIdCalculatedFields.get(newCf.getId()); + List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); boolean found = false; for (CalculatedFieldCtx oldCtx : oldCfList) { @@ -265,10 +266,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!found) { newCfList.add(newCfCtx); } - entityIdCalculatedFields.put(newCf.getId(), newCfList); + entityIdCalculatedFields.put(newCf.getEntityId(), newCfList); + + deleteLinks(oldCfCtx); + addLinks(newCf); + // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) - if (newCfCtx.hasSignificantChanges(oldCfCtx)) { + var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { try { newCfCtx.init(); } catch (Exception e) { @@ -276,11 +282,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); } } - initCf(newCfCtx, callback, true); + initCf(newCfCtx, callback, stateChanges); + } else { + callback.onSuccess(); } } } - } private void onCfDeleted(ComponentLifecycleMsg msg, TbCallback callback) { @@ -290,6 +297,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); } else { + deleteLinks(cfCtx); + EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { @@ -440,4 +449,16 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware () -> true); } + private void addLinks(CalculatedField newCf) { + var newLinks = newCf.getConfiguration().buildCalculatedFieldLinks(tenantId, newCf.getEntityId(), newCf.getId()); + newLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link)); + } + + private void deleteLinks(CalculatedFieldCtx cfCtx) { + var oldCf = cfCtx.getCalculatedField(); + var oldLinks = oldCf.getConfiguration().buildCalculatedFieldLinks(tenantId, oldCf.getEntityId(), oldCf.getId()); + oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); + } + + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c8f3d98f40..9688f35fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -70,20 +70,25 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @AfterStartUp(order = AfterStartUp.CF_READ_CF_SERVICE) public void init() { + //TODO: move to separate place to avoid circular references with the ActorSystemContext (@Lazy for tsSubService) PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); - cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); - calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) - ); - cfs.forEach(cf -> actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf))); + cfs.forEach(cf -> { + calculatedFields.putIfAbsent(cf.getId(), cf); + actorSystemContext.tell(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + }); + calculatedFields.values().forEach(cf -> { + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); + }); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); + cfls.forEach(link -> { + calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link); + actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); + }); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); - cfls.forEach(link -> actorSystemContext.tell(new CalculatedFieldLinkInitMsg(link.getTenantId(), link))); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd8c9b7edd..103997f0c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -136,7 +136,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } }; - private final CalculatedFieldService calculatedFieldService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final CalculatedFieldCache calculatedFieldCache; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java index b3499f3cc4..6aa5c3d4de 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.cache; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import java.util.ArrayList; @@ -88,6 +89,9 @@ public class TenantEntityProfileCache { public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { lock.writeLock().lock(); try { + if(EntityType.DEVICE.equals(profileId.getEntityType())){ + throw new RuntimeException("WTF?"); + } if (mine) { myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c483c6ab5d..c3105d6b57 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -212,12 +212,16 @@ public class CalculatedFieldCtx { return new CalculatedFieldEntityCtxId(tenantId, cfId, entityId); } - public boolean hasSignificantChanges(CalculatedFieldCtx other) { - boolean entityIdChanged = !entityId.equals(other.entityId); + public boolean hasOtherSignificantChanges(CalculatedFieldCtx other) { + boolean expressionChanged = !expression.equals(other.expression); + boolean outputChanged = !output.equals(other.output); + return expressionChanged || outputChanged; + } + + public boolean hasStateChanges(CalculatedFieldCtx other) { boolean typeChanged = !cfType.equals(other.cfType); boolean argumentsChanged = !arguments.equals(other.arguments); - boolean expressionChanged = !expression.equals(other.expression); - return entityIdChanged || typeChanged || argumentsChanged || expressionChanged; + return typeChanged || argumentsChanged; } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index d40bf7b130..7265533aab 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -178,11 +178,11 @@ public class EntityStateSourcingListener { } case TENANT_PROFILE -> { TenantProfile tenantProfile = (TenantProfile) event.getEntity(); - tbClusterService.onTenantProfileDelete(tenantProfile, null); + tbClusterService.onTenantProfileDelete(tenantProfile, TbQueueCallback.EMPTY); } case DEVICE -> { Device device = (Device) event.getEntity(); - tbClusterService.onDeviceDeleted(tenantId, device, null); + tbClusterService.onDeviceDeleted(tenantId, device, TbQueueCallback.EMPTY); } case DEVICE_PROFILE -> { DeviceProfile deviceProfile = (DeviceProfile) event.getEntity(); @@ -190,11 +190,11 @@ public class EntityStateSourcingListener { } case TB_RESOURCE -> { TbResourceInfo tbResource = (TbResourceInfo) event.getEntity(); - tbClusterService.onResourceDeleted(tbResource, null); + tbClusterService.onResourceDeleted(tbResource, TbQueueCallback.EMPTY); } case CALCULATED_FIELD -> { CalculatedField calculatedField = (CalculatedField) event.getEntity(); - tbClusterService.onCalculatedFieldDeleted(calculatedField, null); + tbClusterService.onCalculatedFieldDeleted(calculatedField, TbQueueCallback.EMPTY); } default -> { } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 944e24480e..fa02435ca0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -727,14 +727,19 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void onCalculatedFieldUpdated(CalculatedField calculatedField, CalculatedField oldCalculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), oldCalculatedField == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED)); + onCalculatedFieldLifecycleMsg(msg, callback); } @Override public void onCalculatedFieldDeleted(CalculatedField calculatedField, TbQueueCallback callback) { - var msg = new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED); - broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), callback); + var msg = toProto(new ComponentLifecycleMsg(calculatedField.getTenantId(), calculatedField.getId(), ComponentLifecycleEvent.DELETED)); + onCalculatedFieldLifecycleMsg(msg, callback); + } + + private void onCalculatedFieldLifecycleMsg(ComponentLifecycleMsgProto msg, TbQueueCallback callback) { + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(msg).build(), callback); + broadcastToCore(ToCoreNotificationMsg.newBuilder().setComponentLifecycle(msg).build()); } @Override diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index ee4af20639..a2a26d55e4 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -159,7 +159,8 @@ public final class TbActorMailbox implements TbActorCtx { stopReason = TbActorStopReason.INIT_FAILED; destroy(updateException.getCause()); } catch (Throwable t) { - log.debug("[{}] Failed to process message: {}", selfId, msg, t); + //TODO: revert; + log.error("[{}] Failed to process message: {}", selfId, msg, t); ProcessFailureStrategy strategy = actor.onProcessFailure(msg, t); if (strategy.isStop()) { system.stop(selfId); From 09bd3de984d15c7596bdc7d5152b3c5bc91ec342 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 16:42:31 +0200 Subject: [PATCH 141/438] test fixes --- .../controller/EntityQueryControllerTest.java | 16 ++-------------- .../discovery/HashPartitionServiceTest.java | 1 + .../state/DefaultDeviceStateServiceTest.java | 18 +++++++++--------- .../update/330/device_profile_001_out.json | 6 ++++-- 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 508f66e584..d9652946ab 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -817,26 +817,14 @@ public class EntityQueryControllerTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, null, null); - PageData data = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(1, data.getTotalElements()); - Assert.assertEquals(1, data.getTotalPages()); - Assert.assertEquals(1, data.getData().size()); + findByQueryAndCheck(query, 1); // unnassign dashboard login(TENANT_EMAIL, TENANT_PASSWORD); doDelete("/api/customer/" + savedCustomer.getId().getId().toString() + "/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); login(CUSTOMER_USER_EMAIL, CUSTOMER_USER_PASSWORD); - PageData dataAfterUnassign = - doPostWithTypedResponse("/api/entitiesQuery/find", query, new TypeReference>() { - }); - - Assert.assertEquals(0, dataAfterUnassign.getTotalElements()); - Assert.assertEquals(0, dataAfterUnassign.getTotalPages()); - Assert.assertEquals(0, dataAfterUnassign.getData().size()); + findByQueryAndCheck(query, 0); } private void checkEntitiesByQuery(EntityDataQuery query, int expectedNumOfDevices, String expectedOwnerName, String expectedOwnerType) throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 0a4009e8b2..0d3ae093aa 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -431,6 +431,7 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); ReflectionTestUtils.setField(partitionService, "edgeTopic", "tb.edge"); ReflectionTestUtils.setField(partitionService, "edgePartitions", 10); + ReflectionTestUtils.setField(partitionService, "edqsPartitions", 12); partitionService.init(); partitionService.partitionsInit(); return partitionService; diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java index 9880ec964b..3c3fe2716c 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultDeviceStateServiceTest.java @@ -211,7 +211,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_CONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastConnectTime) @@ -298,7 +298,7 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(LAST_DISCONNECT_TIME) && request.getEntries().get(0).getValue().equals(lastDisconnectTime) @@ -421,13 +421,13 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(lastInactivityTime) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -465,12 +465,12 @@ public class DefaultDeviceStateServiceTest { // THEN then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) )); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(ACTIVITY_STATE) && request.getEntries().get(0).getValue().equals(false) @@ -1002,7 +1002,7 @@ public class DefaultDeviceStateServiceTest { assertThat(actualNotification.isActive()).isFalse(); then(telemetrySubscriptionService).should().saveAttributes(argThat(request -> - request.getTenantId().equals(TenantId.SYS_TENANT_ID) && request.getEntityId().equals(deviceId) && + request.getTenantId().equals(tenantId) && request.getEntityId().equals(deviceId) && request.getScope().equals(AttributeScope.SERVER_SCOPE) && request.getEntries().get(0).getKey().equals(INACTIVITY_ALARM_TIME) && request.getEntries().get(0).getValue().equals(expectedLastInactivityAlarmTime) @@ -1170,7 +1170,7 @@ public class DefaultDeviceStateServiceTest { assertThat(attributeRequestCaptor.getAllValues()).hasSize(2) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { @@ -1179,7 +1179,7 @@ public class DefaultDeviceStateServiceTest { }); }) .anySatisfy(request -> { - assertThat(request.getTenantId()).isEqualTo(TenantId.SYS_TENANT_ID); + assertThat(request.getTenantId()).isEqualTo(tenantId); assertThat(request.getEntityId()).isEqualTo(deviceId); assertThat(request.getScope()).isEqualTo(AttributeScope.SERVER_SCOPE); assertThat(request.getEntries()).singleElement().satisfies(attributeKvEntry -> { diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json index 9a349c6638..29e2241ee9 100644 --- a/application/src/test/resources/update/330/device_profile_001_out.json +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -64,7 +64,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } @@ -103,7 +104,8 @@ "dynamicValue": { "sourceType": null, "sourceAttribute": null, - "inherit": false + "inherit": false, + "resolvedValue" : null } } } From 669bfcbb2255dd970705ebe8f5031d2c625d0e2e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 5 Feb 2025 17:57:18 +0200 Subject: [PATCH 142/438] Added calculated field import/export --- .../calculated-fields-table-config.ts | 37 +++++++++++++- .../calculated-fields-table.component.ts | 5 +- .../import-export/import-dialog.component.ts | 2 +- .../import-export/import-export.service.ts | 48 +++++++++++++++++++ .../shared/models/calculated-field.models.ts | 12 +++-- .../assets/locale/locale.constant-en_US.json | 6 +++ 6 files changed, 104 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index dc7aeffb1d..3ff5f3bc7e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -37,6 +37,7 @@ import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDialogComponent } from './components/public-api'; +import { ImportExportService } from '@shared/import-export/import-export.service'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -55,7 +56,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.translate.instant('calculated-fields.delete-multiple-title', {count}); this.deleteEntitiesContent = () => this.translate.instant('calculated-fields.delete-multiple-text'); this.deleteEntity = id => this.calculatedFieldsService.deleteCalculatedField(id.id); + this.addActionDescriptors = [ + { + name: this.translate.instant('calculated-fields.create'), + icon: 'insert_drive_file', + isEnabled: () => true, + onAction: ($event) => this.getTable().addEntity($event) + }, + { + name: this.translate.instant('calculated-fields.import'), + icon: 'file_upload', + isEnabled: () => true, + onAction: () => this.importCalculatedField() + } + ]; this.defaultSortOrder = {property: 'name', direction: Direction.DESC}; @@ -82,6 +98,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, + onAction: (event$, entity) => this.exportCalculatedField(event$, entity), + }, { name: '', nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), @@ -166,6 +188,19 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); + } + private getDebugConfigLabel(debugSettings: EntityDebugSettings): string { const isDebugActive = this.isDebugActive(debugSettings?.allEnabledUntil); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts index d212dfcbf7..acd7b18000 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table.component.ts @@ -34,6 +34,7 @@ import { CalculatedFieldsTableConfig } from '@home/components/calculated-fields/ import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { TbPopoverService } from '@shared/components/popover.service'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { ImportExportService } from '@shared/import-export/import-export.service'; @Component({ selector: 'tb-calculated-fields-table', @@ -59,6 +60,7 @@ export class CalculatedFieldsTableComponent { private popoverService: TbPopoverService, private cd: ChangeDetectorRef, private renderer: Renderer2, + private importExportService: ImportExportService, private destroyRef: DestroyRef) { effect(() => { @@ -73,7 +75,8 @@ export class CalculatedFieldsTableComponent { this.popoverService, this.destroyRef, this.renderer, - this.entityName() + this.entityName(), + this.importExportService ); this.cd.markForCheck(); } diff --git a/ui-ngx/src/app/shared/import-export/import-dialog.component.ts b/ui-ngx/src/app/shared/import-export/import-dialog.component.ts index e4256229c4..7e46f47978 100644 --- a/ui-ngx/src/app/shared/import-export/import-dialog.component.ts +++ b/ui-ngx/src/app/shared/import-export/import-dialog.component.ts @@ -71,7 +71,7 @@ export class ImportDialogComponent extends DialogComponent, isSingleWidget: boolean, customTitle: string, missingEntityAliases: EntityAliases) => Observable; @@ -116,6 +118,7 @@ export class ImportExportService { private imageService: ImageService, private utils: UtilsService, private itembuffer: ItemBufferService, + private calculatedFieldsService: CalculatedFieldsService, private dialog: MatDialog) { } @@ -171,6 +174,35 @@ export class ImportExportService { ); } + public exportCalculatedField(calculatedFieldId: string): void { + this.calculatedFieldsService.getCalculatedFieldById(calculatedFieldId).subscribe({ + next: (calculatedField) => { + let name = calculatedField.name; + name = name.toLowerCase().replace(/\W/g, '_'); + this.exportToPc(this.prepareCalculatedFieldExport(calculatedField), name); + }, + error: (e) => { + this.handleExportError(e, 'calculated-fields.export-failed-error'); + } + }); + } + + public importCalculatedField(entityId: EntityId): Observable { + return this.openImportDialog('calculated-fields.import', 'calculated-fields.file').pipe( + mergeMap((calculatedField: CalculatedField) => { + if (!this.validateImportedCalculatedField({ entityId, ...calculatedField })) { + this.store.dispatch(new ActionNotificationShow( + {message: this.translate.instant('calculated-fields.invalid-file-error'), + type: 'error'})); + throw new Error('Invalid calculated field file'); + } else { + return this.calculatedFieldsService.saveCalculatedField(this.prepareImport({ entityId, ...calculatedField })); + } + }), + catchError(() => of(null)), + ); + } + public exportDashboard(dashboardId: string) { this.getIncludeResourcesPreference('includeResourcesInExportDashboard').subscribe(includeResources => { this.openExportDialog('dashboard.export', 'dashboard.export-prompt', includeResources).subscribe(result => { @@ -957,6 +989,17 @@ export class ImportExportService { } } + private validateImportedCalculatedField(calculatedField: CalculatedField): boolean { + const { name, configuration, entityId } = calculatedField; + return isNotEmptyStr(name) + && isDefined(configuration) + && isDefined(entityId?.id) + && !!Object.keys(configuration.arguments).length + && isDefined(configuration.expression) + && isDefined(configuration.output) + && isNotEmptyStr(configuration.output.name); + } + private validateImportedImage(image: ImageExportData): boolean { return !(!isNotEmptyStr(image.data) || !isNotEmptyStr(image.title) @@ -1209,6 +1252,11 @@ export class ImportExportService { return profile; } + private prepareCalculatedFieldExport(calculatedField: CalculatedField): CalculatedField { + delete calculatedField.entityId; + return this.prepareExport(calculatedField); + } + private prepareExport(data: any): any { const exportedData = deepClone(data); if (isDefined(exportedData.id)) { diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index fac1e9f942..3a86b12018 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -15,16 +15,15 @@ /// import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType } from '@shared/models/alias.models'; -export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId { +export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId, ExportableEntity { debugSettings?: EntityDebugSettings; - externalId?: string; configuration: CalculatedFieldConfiguration; type: CalculatedFieldType; entityId: EntityId; @@ -46,6 +45,13 @@ export interface CalculatedFieldConfiguration { type: CalculatedFieldType; expression: string; arguments: Record; + output: CalculatedFieldOutput; +} + +export interface CalculatedFieldOutput { + type: OutputType; + name: string; + scope?: AttributeScope; } export enum ArgumentEntityType { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cb6f124b1d..469bc9a00f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1043,6 +1043,12 @@ "asset-name": "Asset name", "timeseries": "Time series", "output": "Output", + "create": "Create new calculated field", + "file": "Calculated field file", + "invalid-file-error": "Invalid file format. Please make sure the file is a valid JSON file.", + "import": "Import calculated field", + "export": "Export calculated field", + "export-failed-error": "Unable to export calculated field: {{error}}", "output-type": "Output type", "delete-title": "Are you sure you want to delete the calculated field '{{title}}'?", "delete-text": "Be careful, after the confirmation the calculated field and all related data will become unrecoverable.", From f128f49d44edbdafbe6927b3137a3106cdc33bf7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 5 Feb 2025 18:57:23 +0200 Subject: [PATCH 143/438] test fixes --- .../service/entitiy/EntityServiceTest.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index fab4cba3f5..a559eca6ce 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -1178,8 +1178,7 @@ public class EntityServiceTest extends AbstractControllerTest { EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); - PageData result = searchEntities(query); - assertEquals(1, result.getTotalElements()); + PageData result = findByQueryAndCheck(query, 1); String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); assertThat(deviceName).isEqualTo(devices.get(0).getName()); @@ -1240,11 +1239,8 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setRootEntity(asset.getId()); EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); - PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); - long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); - - Assert.assertEquals(relationsCnt, relationsResult.getData().size()); - Assert.assertEquals(relationsCnt, relationsResultCnt); + findByQueryAndCheck(customer.getId(), query, relationsCnt); + countByQueryAndCheck(customer.getId(), query, relationsCnt); } } @@ -1497,9 +1493,6 @@ public class EntityServiceTest extends AbstractControllerTest { assertThat(tenantResultName).isEqualTo(TEST_CUSTOMER_NAME); } - private PageData searchEntities(EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); - } private EntityDataQuery createDeviceSearchQuery(String deviceField, StringOperation operation, String searchQuery) { DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); @@ -1593,7 +1586,7 @@ public class EntityServiceTest extends AbstractControllerTest { .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); } List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceTemperatures, loadedTemperatures); + assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); @@ -1614,8 +1607,7 @@ public class EntityServiceTest extends AbstractControllerTest { entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures); - + assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures); } deviceService.deleteDevicesByTenantId(tenantId); } From 2214bf8ef23a6cb824926d174d92303d2bc9da54 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:47:52 +0200 Subject: [PATCH 144/438] Calculated Fields Debug Events dialog basic implementation --- .../server/common/data/event/EventFilter.java | 3 +- .../calculated-fields-table-config.ts | 32 ++++++- ...lculated-field-debug-dialog.component.html | 47 ++++++++++ ...calculated-field-debug-dialog.component.ts | 53 +++++++++++ .../calculated-field-dialog.component.html | 1 + .../calculated-field-dialog.component.ts | 8 ++ .../components/public-api.ts | 1 + .../entity-debug-settings-button.component.ts | 6 +- ...entity-debug-settings-panel.component.html | 40 ++++++--- .../entity-debug-settings-panel.component.ts | 5 +- .../components/event/event-table-config.ts | 89 +++++++++++++++++++ .../home/components/home-components.module.ts | 5 ++ .../shared/models/calculated-field.models.ts | 14 ++- ui-ngx/src/app/shared/models/entity.models.ts | 9 ++ ui-ngx/src/app/shared/models/event.models.ts | 16 +++- .../assets/locale/locale.constant-en_US.json | 5 ++ 16 files changed, 310 insertions(+), 24 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 6d2a110cf7..4c9791e3fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 3ff5f3bc7e..c5e455583d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -35,8 +35,12 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap } from 'rxjs/operators'; -import { CalculatedField, CalculatedFieldDialogData } from '@shared/models/calculated-field.models'; -import { CalculatedFieldDialogComponent } from './components/public-api'; +import { + CalculatedField, + CalculatedFieldDebugDialogData, + CalculatedFieldDialogData +} from '@shared/models/calculated-field.models'; +import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -46,6 +50,14 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog({...this.additionalDebugActionConfig.data, id }), + }; const { viewContainerRef } = this.getTable(); if ($event) { $event.stopPropagation(); @@ -140,6 +156,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data + }) + .afterClosed() + .subscribe(); + } + private exportCalculatedField($event: Event, calculatedField: CalculatedField): void { if ($event) { $event.stopPropagation(); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html new file mode 100644 index 0000000000..1b61a9da4a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -0,0 +1,47 @@ + +
+ +

{{ 'calculated-fields.debugging' | translate}}

+ + + +
+ +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts new file mode 100644 index 0000000000..8efb21dbf2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -0,0 +1,53 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, Inject, ViewChild } from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { DebugEventType, EventType } from '@shared/models/event.models'; +import { EventTableComponent } from '@home/components/event/event-table.component'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-debug-dialog', + templateUrl: './calculated-field-debug-dialog.component.html', +}) +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; + + readonly DebugEventType = DebugEventType; + readonly debugEventTypes = DebugEventType; + readonly EventType = EventType; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, + protected dialogRef: MatDialogRef) { + super(store, router, dialogRef); + } + + ngAfterViewInit(): void { + this.eventsTable.entitiesTable.updateData(); + } + + cancel(): void { + this.dialogRef.close(null); + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 37e96cfd91..ac60ae8178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -51,6 +51,7 @@ [class.mb-5]="fieldFormGroup.get('name').errors && fieldFormGroup.get('name').touched" [entityLabel]="'debug-settings.calculated-field' | translate" [debugLimitsConfiguration]="data.debugLimitsConfiguration" + [additionalActionConfig]="additionalDebugActionConfig" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 55fc299475..3575223764 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -65,6 +65,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent Object.keys(argumentsObj)) ); + additionalDebugActionConfig = this.data.value?.id ? { + ...this.data.additionalDebugActionConfig, + action: () => this.data.additionalDebugActionConfig.action({ + ...this.data.additionalDebugActionConfig.data, + id: this.data.value.id, + }), + } : null; + readonly OutputTypeTranslations = OutputTypeTranslations; readonly OutputType = OutputType; readonly AttributeScope = AttributeScope; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index bc89e4dc6f..78b8862c2e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -17,3 +17,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; +export * from './debug-dialog/calculated-field-debug-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts index 597fd79b8e..7e06568489 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.ts @@ -32,7 +32,7 @@ import { EntityDebugSettingsPanelComponent } from './entity-debug-settings-panel import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { BehaviorSubject, of, shareReplay, timer } from 'rxjs'; import { SECOND, MINUTE } from '@shared/models/time/time.models'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { map, switchMap, takeWhile } from 'rxjs/operators'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { AppState } from '@core/core.state'; @@ -61,6 +61,7 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor @Input() debugLimitsConfiguration: string; @Input() entityLabel: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; debugSettingsFormGroup = this.fb.group({ failuresEnabled: [false], @@ -133,7 +134,8 @@ export class EntityDebugSettingsButtonComponent implements ControlValueAccessor ...debugSettings, maxDebugModeDuration: this.maxDebugModeDuration, debugLimitsConfiguration: this.debugLimitsConfiguration, - entityLabel: this.entityLabel + entityLabel: this.entityLabel, + additionalActionConfig: this.additionalActionConfig, }, {}, {}, {}, true); diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index 7576d4b2fe..dcc3355890 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -48,20 +48,32 @@ -
- - +
+
+ @if (additionalActionConfig) { + + } +
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts index 72748e27e2..9e8ac1ded6 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.ts @@ -21,7 +21,7 @@ import { Component, EventEmitter, Input, - OnInit + OnInit, } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { TbPopoverComponent } from '@shared/components/popover.component'; @@ -32,7 +32,7 @@ import { SECOND } from '@shared/models/time/time.models'; import { DurationLeftPipe } from '@shared/pipe/duration-left.pipe'; import { of, shareReplay, timer } from 'rxjs'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { EntityDebugSettings } from '@shared/models/entity.models'; +import { AdditionalDebugActionConfig, EntityDebugSettings } from '@shared/models/entity.models'; import { distinctUntilChanged, map, startWith, switchMap, takeWhile } from 'rxjs/operators'; @Component({ @@ -54,6 +54,7 @@ export class EntityDebugSettingsPanelComponent extends PageComponent implements @Input() allEnabledUntil = 0; @Input() maxDebugModeDuration: number; @Input() debugLimitsConfiguration: string; + @Input() additionalActionConfig: AdditionalDebugActionConfig; onFailuresControl = this.fb.control(false); debugAllControl = this.fb.control(false); diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 953d1d17c1..8de4a9f256 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -355,6 +355,86 @@ export class EventTableConfig extends EntityTableConfig { '48px') ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.columns[0].width = '160px'; + this.columns.push( + new EntityTableColumn('entityId', 'event.entity-id', '150px', + (entity) => entity.body.entityId, + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-entity-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.entityId, + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageId', 'event.message-id', '150px', + (entity) => entity.body.msgId ?? '', + () => ({padding: '0 12px 0 0'}), + false, + () => ({padding: '0 12px 0 0'}), + () => undefined, + false, + { + name: this.translate.instant('event.copy-message-id'), + icon: 'content_paste', + style: { + padding: '4px', + 'font-size': '16px', + color: 'rgba(0,0,0,.87)' + }, + isEnabled: () => true, + onAction: ($event, entity) => entity.body.msgId ?? '', + type: CellActionDescriptorType.COPY_BUTTON + } + ), + new EntityTableColumn('messageType', 'event.message-type', '150px', + (entity) => entity.body.msgType ?? '', + () => ({padding: '0 12px 0 0'}), + false + ), + new EntityActionTableColumn('arguments', 'event.arguments', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.arguments !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.arguments, + 'event.arguments', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('result', 'event.result', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.result !== undefined, + onAction: ($event, entity) => this.showContent($event, entity.body.result, + 'event.result', ContentType.JSON, true) + }, + '100px' + ), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '100px' + ) + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -446,6 +526,15 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.filterColumns.push( + {key: 'entityId', title: 'event.entity-id'}, + {key: 'messageId', title: 'event.message-id'}, + {key: 'messageType', title: 'event.message-type'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f60ea15407..b7e35c5406 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -195,6 +195,9 @@ import { import { CalculatedFieldArgumentPanelComponent } from '@home/components/calculated-fields/components/panel/calculated-field-argument-panel.component'; +import { + CalculatedFieldDebugDialogComponent +} from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; @NgModule({ declarations: @@ -343,6 +346,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], imports: [ CommonModule, @@ -485,6 +489,7 @@ import { CalculatedFieldDialogComponent, CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, + CalculatedFieldDebugDialogComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 3a86b12018..31dca236a4 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -14,7 +14,12 @@ /// limitations under the License. /// -import { EntityDebugSettings, HasTenantId, HasVersion } from '@shared/models/entity.models'; +import { + AdditionalDebugActionConfig, + EntityDebugSettings, + HasTenantId, + HasVersion +} from '@shared/models/entity.models'; import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; import { EntityId } from '@shared/models/id/entity-id'; @@ -128,6 +133,13 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; + additionalDebugActionConfig: AdditionalDebugActionConfig; +} + +export interface CalculatedFieldDebugDialogData { + id?: CalculatedFieldId; + entityId: EntityId; + tenantId: string; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 9934da65aa..db00955340 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,6 +21,7 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; +import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -203,4 +204,12 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; + +export interface AdditionalDebugActionConfig { + action?: (data?: AdditionalDebugActionConfigData) => void; + title: string; + data: AdditionalDebugActionConfigData; +} + export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index a1abb8d15c..8a5009f758 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -29,7 +29,8 @@ export enum EventType { export enum DebugEventType { DEBUG_RULE_NODE = 'DEBUG_RULE_NODE', - DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN' + DEBUG_RULE_CHAIN = 'DEBUG_RULE_CHAIN', + DEBUG_CALCULATED_FIELD = 'DEBUG_CALCULATED_FIELD' } export const eventTypeTranslations = new Map( @@ -39,6 +40,7 @@ export const eventTypeTranslations = new Map [EventType.STATS, 'event.type-stats'], [DebugEventType.DEBUG_RULE_NODE, 'event.type-debug-rule-node'], [DebugEventType.DEBUG_RULE_CHAIN, 'event.type-debug-rule-chain'], + [DebugEventType.DEBUG_CALCULATED_FIELD, 'event.type-debug-calculated-field'], ] ); @@ -80,7 +82,7 @@ export interface DebugRuleChainEventBody extends BaseEventBody { error?: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody & CalculatedFieldEventBody; export interface Event extends BaseData { tenantId: TenantId; @@ -90,6 +92,16 @@ export interface Event extends BaseData { body: EventBody; } +export interface CalculatedFieldEventBody extends BaseFilterEventBody { + calculatedFieldId: string; + entityId: string; + entityType: EntityType; + arguments: string, + result: string, + msgId: string; + msgType: string; +} + export interface BaseFilterEventBody { server?: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 469bc9a00f..9bdf533fef 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1014,6 +1014,7 @@ "script": "Script" }, "arguments": "Arguments", + "debugging": "Calculated field debugging", "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", @@ -1026,6 +1027,7 @@ "argument-customer": "Customer", "argument-tenant": "Current tenant", "argument-type": "Argument type", + "see-debug-events": "See debug events", "attribute": "Attribute", "timeseries-key": "Time series key", "device-name": "Device name", @@ -2710,6 +2712,9 @@ "type-stats": "Statistics", "type-debug-rule-node": "Debug", "type-debug-rule-chain": "Debug", + "type-debug-calculated-field": "Debug", + "arguments": "Arguments", + "result": "Result", "no-events-prompt": "No events found", "error": "Error", "alarm": "Alarm", From 7ad8913b16b6d2ac0b99e57c0a7fa1ea520b4f79 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:52:29 +0200 Subject: [PATCH 145/438] refactoring --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 1b61a9da4a..715cf0572d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data?.id" + [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" /> From c8b7b1c3a243ef021080014cc54f7a83d5ca44db Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 15:53:55 +0200 Subject: [PATCH 146/438] refactoring --- .../app/modules/home/components/event/event-table-config.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 8de4a9f256..da391d3ecb 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '150px', - (entity) => entity.body.msgId ?? '', + (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -394,12 +394,12 @@ export class EventTableConfig extends EntityTableConfig { color: 'rgba(0,0,0,.87)' }, isEnabled: () => true, - onAction: ($event, entity) => entity.body.msgId ?? '', + onAction: ($event, entity) => entity.body.msgId, type: CellActionDescriptorType.COPY_BUTTON } ), new EntityTableColumn('messageType', 'event.message-type', '150px', - (entity) => entity.body.msgType ?? '', + (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false ), From 5354eaa80b7ffceece43aa7ea3d8d0e52e875aa1 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 16:10:26 +0200 Subject: [PATCH 147/438] Adjusted sizing --- .../calculated-field-debug-dialog.component.html | 2 +- .../home/components/event/event-table-config.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 715cf0572d..25f2a11f16 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index da391d3ecb..ae98560969 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -358,7 +358,7 @@ export class EventTableConfig extends EntityTableConfig { case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '160px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '150px', + new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => entity.body.entityId, () => ({padding: '0 12px 0 0'}), false, @@ -378,7 +378,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '150px', + new EntityTableColumn('messageId', 'event.message-id', '85px', (entity) => entity.body.msgId, () => ({padding: '0 12px 0 0'}), false, @@ -398,7 +398,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageType', 'event.message-type', '150px', + new EntityTableColumn('messageType', 'event.message-type', '100px', (entity) => entity.body.msgType, () => ({padding: '0 12px 0 0'}), false @@ -411,7 +411,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.arguments, 'event.arguments', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('result', 'event.result', { @@ -421,7 +421,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.result, 'event.result', ContentType.JSON, true) }, - '100px' + '48px' ), new EntityActionTableColumn('error', 'event.error', { @@ -431,7 +431,7 @@ export class EventTableConfig extends EntityTableConfig { onAction: ($event, entity) => this.showContent($event, entity.body.error, 'event.error') }, - '100px' + '48px' ) ); break; From 7fcd948071ffbefb0ff654758e58346a05150546 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 16:11:58 +0200 Subject: [PATCH 148/438] fixed error when no telemetry in db --- .../cf/DefaultCalculatedFieldCache.java | 3 ++- ...efaultCalculatedFieldExecutionService.java | 5 ++++- .../ctx/state/BaseCalculatedFieldState.java | 2 +- .../cf/ctx/state/RocksDBStateService.java | 19 +++++++++++++++---- .../ctx/state/SingleValueArgumentEntry.java | 9 ++++++++- .../state/SingleValueArgumentEntryTest.java | 4 ++++ .../server/common/data/event/EventFilter.java | 3 ++- 7 files changed, 36 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 9688f35fef..8fffa0029c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.dao.cf.CalculatedFieldService; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.dao.usagerecord.ApiLimitService; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.Collections; @@ -117,6 +117,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService, apiLimitService); + ctx.init(); calculatedFieldsCtx.put(calculatedFieldId, ctx); log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 103997f0c7..43e85d87e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; @@ -68,7 +69,6 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; @@ -447,6 +447,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); + if (StringUtils.isBlank(defaultValue)) { + return new StringDataEntry(key, null); + } if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java index 86d83a1f70..f1ce8038c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java @@ -52,7 +52,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState { ArgumentEntry newEntry = entry.getValue(); ArgumentEntry existingEntry = arguments.get(key); - if (existingEntry == null) { + if (existingEntry == null || existingEntry == SingleValueArgumentEntry.EMPTY || existingEntry == TsRollingArgumentEntry.EMPTY) { validateNewEntry(newEntry); arguments.put(key, newEntry); stateUpdated = true; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java index 8a6a5c9cb7..b2e33e1705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -63,7 +63,7 @@ public class RocksDBStateService implements CalculatedFieldStateService { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { - rocksDBService.put(toProto(stateId), toProto(stateId, state)); + rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); } @@ -111,8 +111,11 @@ public class RocksDBStateService implements CalculatedFieldStateService { private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName) - .setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + .setArgName(argName); + + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); @@ -122,7 +125,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } return builder.build(); } @@ -151,6 +156,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } TsValueProto tsValueProto = proto.getValue(); long ts = tsValueProto.getTs(); BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); @@ -158,6 +166,9 @@ public class RocksDBStateService implements CalculatedFieldStateService { } private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } TreeMap tsRecords = new TreeMap<>(); proto.getTsValueList().forEach(tsValueProto -> { BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java index 0832e53e5e..d7e5ddd017 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java @@ -94,10 +94,17 @@ public class SingleValueArgumentEntry implements ArgumentEntry { Long newVersion = singleValueEntry.getVersion(); if (newVersion == null || this.version == null || newVersion > this.version) { this.ts = singleValueEntry.getTs(); - this.kvEntryValue = singleValueEntry.getKvEntryValue(); this.version = newVersion; + + // TODO: should we persist updated ts and version values? + BasicKvEntry newValue = singleValueEntry.getKvEntryValue(); + if (this.kvEntryValue.getValue().equals(newValue.getValue())) { + return false; + } + this.kvEntryValue = singleValueEntry.getKvEntryValue(); return true; } + } else { throw new IllegalArgumentException("Unsupported argument entry type for single value argument entry: " + entry.getType()); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 203d7b3d71..13651e852d 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -69,4 +69,8 @@ public class SingleValueArgumentEntryTest { assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 18L), 234L))).isFalse(); } + @Test + void testUpdateEntryWhenValueWasNotChanged() { + assertThat(entry.updateEntry(new SingleValueArgumentEntry(ts + 18, new LongDataEntry("key", 11L), 237L))).isFalse(); + } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 6d2a110cf7..4c9791e3fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -29,7 +29,8 @@ import io.swagger.v3.oas.annotations.media.Schema; @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), - @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") + @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS"), + @JsonSubTypes.Type(value = CalculatedFieldDebugEventFilter.class, name = "DEBUG_CALCULATED_FIELD") }) public interface EventFilter { From ff647aedc7c95c205fd4d27b52867b140504f2d4 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:01:33 +0200 Subject: [PATCH 149/438] Improved context passing --- .../calculated-fields-table-config.ts | 17 +++++++++-------- .../dialog/calculated-field-dialog.component.ts | 5 +---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +----- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c5e455583d..026c249159 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -42,6 +42,7 @@ import { } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -52,11 +53,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -140,7 +137,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog({...this.additionalDebugActionConfig.data, id }), + action: () => this.openDebugDialog(id) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -206,11 +203,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data + data: { + tenantId: this.tenantId, + entityId: this.entityId, + id + } }) .afterClosed() .subscribe(); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 3575223764..d5bf243430 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -67,10 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action({ - ...this.data.additionalDebugActionConfig.data, - id: this.data.value.id, - }), + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index db00955340..b9d2402850 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -21,7 +21,6 @@ import { DeviceCredentialMQTTBasic } from '@shared/models/device.models'; import { Lwm2mSecurityConfigModels } from '@shared/models/lwm2m-security-config.models'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainMetaData } from '@shared/models/rule-chain.models'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; export interface EntityInfo { name?: string; @@ -204,12 +203,9 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export type AdditionalDebugActionConfigData = CalculatedFieldDebugDialogData; - export interface AdditionalDebugActionConfig { - action?: (data?: AdditionalDebugActionConfigData) => void; + action?: (id?: EntityId) => void; title: string; - data: AdditionalDebugActionConfigData; } export type VersionedEntity = EntityInfoData & HasVersion | RuleChainMetaData; From ebab88ac6a06d04d3141c53107b927cd1257bcef Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 6 Feb 2025 17:06:21 +0200 Subject: [PATCH 150/438] added logs --- ...CalculatedFieldEntityMessageProcessor.java | 7 +++++ ...alculatedFieldManagerMessageProcessor.java | 28 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 309dfde2e9..544dea44c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -83,12 +83,15 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); } public void process(EntityInitCalculatedFieldMsg msg) { + log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); var cfCtx = msg.getCtx(); if (msg.isForceReinit()) { + log.info("Force reinitialization of CF: [{}].", cfCtx.getCfId()); states.remove(cfCtx.getCfId()); } var cfState = getOrInitState(cfCtx); @@ -96,6 +99,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldEntityDeleteMsg msg) { + log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); @@ -110,6 +114,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldTelemetryMsg msg) { + log.info("[{}] Processing CF telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var numberOfCallbacks = CALLBACKS_PER_CF * (msg.getEntityIdFields().size() + msg.getProfileIdFields().size()); MultipleTbCallback callback = new MultipleTbCallback(numberOfCallbacks, msg.getCallback()); @@ -124,6 +129,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityCalculatedFieldLinkedTelemetryMsg msg) { + log.info("[{}] Processing CF link telemetry msg.", msg.getEntityId()); var proto = msg.getProto(); var ctx = msg.getCtx(); var callback = new MultipleTbCallback(CALLBACKS_PER_CF, msg.getCallback()); @@ -169,6 +175,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) { if (newArgValues.isEmpty()) { + log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = getOrInitState(ctx); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index c219544a91..7b03ed2938 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -91,11 +91,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onFieldInitMsg(CalculatedFieldInitMsg msg) { + log.info("[{}] Processing CF init message.", msg.getCf().getId()); var cf = msg.getCf(); var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -108,6 +110,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onLinkInitMsg(CalculatedFieldLinkInitMsg msg) { + log.info("[{}] Processing CF link init message for entity [{}].", msg.getLink().getCalculatedFieldId(), msg.getLink().getEntityId()); var link = msg.getLink(); // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) @@ -121,6 +124,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (calculatedField != null) { msg.getState().setRequiredArguments(calculatedField.getArgNames()); + log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); @@ -128,6 +132,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onEntityLifecycleMsg(CalculatedFieldEntityLifecycleMsg msg) { + log.info("Processing entity lifecycle event: [{}] for entity: [{}]", msg.getData().getEvent(), msg.getData().getEntityId()); var entityType = msg.getData().getEntityId().getEntityType(); var event = msg.getData().getEvent(); switch (entityType) { @@ -207,6 +212,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { cfEntityCache.evict(tenantId, msg.getEntityId()); + log.info("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); } @@ -225,6 +231,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware try { cfCtx.init(); } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", cf.getId(), e); if (DebugModeUtil.isDebugAllAvailable(cf)) { systemContext.persistCalculatedFieldDebugEvent(cf.getTenantId(), cf.getId(), cf.getEntityId(), null, null, null, null, e); } @@ -251,6 +258,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -318,12 +333,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onTelemetryMsg(CalculatedFieldTelemetryMsg msg) { EntityId entityId = msg.getEntityId(); + log.info("Received telemetry msg from entity [{}]", entityId); // 2 = 1 for CF processing + 1 for links processing MultipleTbCallback callback = new MultipleTbCallback(2, msg.getCallback()); // process all cfs related to entity, or it's profile; var entityIdFields = getCalculatedFieldsByEntityId(entityId); var profileIdFields = getCalculatedFieldsByEntityId(getProfileId(tenantId, entityId)); if (!entityIdFields.isEmpty() || !profileIdFields.isEmpty()) { + log.info("Pushing telemetry msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityCalculatedFieldTelemetryMsg(msg, entityIdFields, profileIdFields, callback)); } else { callback.onSuccess(); @@ -340,6 +357,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void onLinkedTelemetryMsg(CalculatedFieldLinkedTelemetryMsg msg) { EntityId sourceEntityId = msg.getEntityId(); + log.info("Received linked telemetry msg from entity [{}]", sourceEntityId); var proto = msg.getProto(); var linksList = proto.getLinksList(); for (var linkProto : linksList) { @@ -353,12 +371,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware if (!entityIds.isEmpty()) { MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); - entityIds.forEach(entityId -> getOrCreateActor(entityId).tell(newMsg)); + entityIds.forEach(entityId -> { + log.info("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + }); } else { msg.getCallback().onSuccess(); } } else { - // push the message to specific entity; + log.info("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); getOrCreateActor(targetEntityId).tell(newMsg); } @@ -423,10 +444,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void deleteCfForEntity(EntityId entityId, CalculatedFieldId cfId, TbCallback callback) { + log.info("Pushing delete CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new CalculatedFieldEntityDeleteMsg(tenantId, cfId, callback)); } private void initCfForEntity(EntityId entityId, CalculatedFieldCtx cfCtx, boolean forceStateReinit, TbCallback callback) { + log.info("Pushing entity init CF msg to specific actor [{}]", entityId); getOrCreateActor(entityId).tell(new EntityInitCalculatedFieldMsg(tenantId, cfCtx, callback, forceStateReinit)); } @@ -460,5 +483,4 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } - } From dfb22314e4783af3cdacc0d8f490ffe6741e0474 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:04 +0200 Subject: [PATCH 151/438] Improved styling --- ...lculated-field-debug-dialog.component.html | 4 ++-- ...lculated-field-debug-dialog.component.scss | 24 +++++++++++++++++++ ...calculated-field-debug-dialog.component.ts | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 25f2a11f16..6166765bc2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
implements AfterViewInit { From 0c226afbc8177968836974ecfb0572cffd59ea78 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:13:45 +0200 Subject: [PATCH 152/438] class order --- .../debug-dialog/calculated-field-debug-dialog.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 6166765bc2..3d92ba5454 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
Date: Thu, 6 Feb 2025 17:14:16 +0200 Subject: [PATCH 153/438] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.html | 2 +- .../debug-dialog/calculated-field-debug-dialog.component.scss | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 3d92ba5454..91da675fea 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 19bf072b11..1e33c9371e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,6 +16,7 @@ :host { .debug-dialog-container { height: 77vh; + min-width: 90vw; .debug-dialog-content { border-radius: 0; From c167c6969fa82b8e83a17c6fe7627e25c9786e25 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:17:10 +0200 Subject: [PATCH 154/438] Improved styling --- .../debug-dialog/calculated-field-debug-dialog.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 1e33c9371e..23aa4070d7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,7 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 90vw; + min-width: 80vw; .debug-dialog-content { border-radius: 0; From bee6375572e42f6f39e6db4fb45e0647a38efd19 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 6 Feb 2025 17:31:59 +0200 Subject: [PATCH 155/438] Improved styling --- .../modules/home/components/event/event-table-config.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index ae98560969..a937e42bb8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -356,10 +356,11 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_CALCULATED_FIELD: - this.columns[0].width = '160px'; + this.columns[0].width = '80px'; + this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => entity.body.entityId, + (entity) => `${entity.body.entityId.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -379,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => entity.body.msgId, + (entity) => `${entity.body.msgId?.substring(0, 6)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), From c582740175ce95814cc80e458505f2b181c29acc Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 09:56:42 +0200 Subject: [PATCH 156/438] added endpoint to test script --- .../controller/CalculatedFieldController.java | 84 +++++++++++++++++++ .../state/CalculatedFieldScriptEngine.java | 3 + .../CalculatedFieldTbelScriptEngine.java | 7 ++ 3 files changed, 94 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 96a97aeeff..435e46069d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; @@ -29,6 +32,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -38,16 +43,27 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldScriptEngine; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngine; import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + import static org.thingsboard.server.controller.ControllerConstants.CF_TEXT_SEARCH_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; @@ -59,9 +75,29 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI public class CalculatedFieldController extends BaseController { private final TbCalculatedFieldService tbCalculatedFieldService; + private final TbelInvokeService tbelInvokeService; public static final String CALCULATED_FIELD_ID = "calculatedFieldId"; + public static final int TIMEOUT = 20; + + private static final String TEST_SCRIPT_EXPRESSION = "Execute the Script expression and return the result. The format of request: \n\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + + " \"argNames\": [\"temperature\", \"humidity\"],\n" + + " \"arguments\": {\n" + + " \"temperature\": {\n" + + " \"14327856345\": 22.4,\n" + + " \"14327857298\": 21.9,\n" + + " \"14327857510\": 22.0\n" + + " },\n" + + " \"humidity\": 42\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + "\n\n Expected result JSON contains \"output\" and \"error\"."; + @ApiOperation(value = "Create Or Update Calculated Field (saveCalculatedField)", notes = "Creates or Updates the Calculated Field. When creating calculated field, platform generates Calculated Field Id as " + UUID_WIKI_LINK + "The newly created Calculated Field Id will be present in the response. " + @@ -128,4 +164,52 @@ public class CalculatedFieldController extends BaseController { tbCalculatedFieldService.delete(calculatedField, getCurrentUser()); } + @ApiOperation(value = "Test Script expression", + notes = TEST_SCRIPT_EXPRESSION + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/calculatedField/testScript", method = RequestMethod.POST) + @ResponseBody + public JsonNode testScript( + @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") + @RequestBody JsonNode inputParams) { + String expression = inputParams.get("expression").asText(); + String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); + Map arguments = Objects.requireNonNullElse( + JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { + }), + Collections.emptyMap() + ); + + String output = ""; + String errorText = ""; + + try { + if (tbelInvokeService == null) { + throw new IllegalArgumentException("TBEL script engine is disabled!"); + } + + CalculatedFieldScriptEngine calculatedFieldScriptEngine = new CalculatedFieldTbelScriptEngine( + getTenantId(), + tbelInvokeService, + expression, + argNames + ); + + Object[] args = Arrays.stream(argNames) + .map(arguments::get) + .toArray(); + + JsonNode json = calculatedFieldScriptEngine.executeJsonAsync(args).get(TIMEOUT, TimeUnit.SECONDS); + output = JacksonUtil.toString(json); + } catch (Exception e) { + log.error("Error evaluating expression", e); + errorText = e.getMessage(); + } + + ObjectNode result = JacksonUtil.newObjectNode(); + result.put("output", output); + result.put("error", errorText); + return result; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java index 779f52c5d6..6bea6ce705 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldScriptEngine.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.ListenableFuture; import java.util.Map; @@ -25,6 +26,8 @@ public interface CalculatedFieldScriptEngine { ListenableFuture> executeToMapAsync(Object[] args); + ListenableFuture executeJsonAsync(Object[] args); + ListenableFuture> executeToMapTransform(Object result); void destroy(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 7ac032573f..9e05e05970 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.id.TenantId; @@ -74,6 +76,11 @@ public class CalculatedFieldTbelScriptEngine implements CalculatedFieldScriptEng return Futures.transformAsync(executeScriptAsync(args), this::executeToMapTransform, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture executeJsonAsync(Object[] args) { + return Futures.transform(executeScriptAsync(args), JacksonUtil::valueToTree, MoreExecutors.directExecutor()); + } + @Override public ListenableFuture> executeToMapTransform(Object result) { if (result instanceof Map) { From c27c82b80d5943b169b6bbc37715cb1d0fe8790e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:01:02 +0200 Subject: [PATCH 157/438] updated test expression endpoint --- .../server/controller/CalculatedFieldController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java index 435e46069d..5c8b40dcad 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CalculatedFieldController.java @@ -48,7 +48,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldTbelScriptEngi import org.thingsboard.server.service.entitiy.cf.TbCalculatedFieldService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -85,7 +85,6 @@ public class CalculatedFieldController extends BaseController { + MARKDOWN_CODE_BLOCK_START + "{\n" + " \"expression\": \"var temp = 0; foreach(element: temperature.entrySet()) { temp += element.getValue(); } var avgTemperature = temp / temperature.size(); var adjustedTemperature = avgTemperature + 0.1 * humidity; return { \\\"adjustedTemperature\\\": adjustedTemperature };\",\n" + - " \"argNames\": [\"temperature\", \"humidity\"],\n" + " \"arguments\": {\n" + " \"temperature\": {\n" + " \"14327856345\": 22.4,\n" + @@ -173,12 +172,12 @@ public class CalculatedFieldController extends BaseController { @io.swagger.v3.oas.annotations.parameters.RequestBody(description = "Test calculated field TBEL expression.") @RequestBody JsonNode inputParams) { String expression = inputParams.get("expression").asText(); - String[] argNames = JacksonUtil.treeToValue(inputParams.get("argNames"), String[].class); Map arguments = Objects.requireNonNullElse( JacksonUtil.convertValue(inputParams.get("arguments"), new TypeReference>() { }), Collections.emptyMap() ); + ArrayList argNames = new ArrayList<>(arguments.keySet()); String output = ""; String errorText = ""; @@ -192,10 +191,10 @@ public class CalculatedFieldController extends BaseController { getTenantId(), tbelInvokeService, expression, - argNames + argNames.toArray(String[]::new) ); - Object[] args = Arrays.stream(argNames) + Object[] args = argNames.stream() .map(arguments::get) .toArray(); From fe4d2ba49759f8dfcf9313e4777bce69235c7c7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 14:24:39 +0200 Subject: [PATCH 158/438] added tests --- ...alculatedFieldManagerMessageProcessor.java | 10 +- .../cf/CalculatedFieldIntegrationTest.java | 266 ++++++++++++++++++ .../CalculatedFieldConfiguration.java | 2 + 3 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 7b03ed2938..6a6edae768 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -258,14 +258,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); - try { - newCfCtx.init(); - } catch (Exception e) { - log.debug("[{}] Failed to initialize CF context.", newCf.getId(), e); - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -289,7 +281,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // We use copy on write lists to safely pass the reference to another actor for the iteration. // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); - if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { + if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { try { newCfCtx.init(); } catch (Exception e) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java new file mode 100644 index 0000000000..fda0724b1b --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cf; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.cf.configuration.Argument; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; +import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; +import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.controller.CalculatedFieldControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@DaoSqlTest +public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + + @BeforeEach + void setUp() throws Exception { + loginTenantAdmin(); + } + + @Test + public void testSimpleCalculatedField() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); + + JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + + config.setArguments(Map.of("T", argument)); + + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + + // update telemetry -> recalculate state + JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + + Thread.sleep(300); + + ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + assertThat(fahrenheitTempAfterUpdate).isNotNull(); + assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF output -> perform calculation with updated output + Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); + savedOutput.setType(OutputType.ATTRIBUTES); + savedOutput.setScope(AttributeScope.SERVER_SCOPE); + savedOutput.setName("temperatureF"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + + // update CF argument -> perform calculation with new argument + + Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); + savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateArg).isNotNull(); + assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + + // update CF expression -> perform calculation with new expression + savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); + savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); + + Thread.sleep(300); + + ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + assertThat(temperatureFAfterUpdateExpression).isNotNull(); + assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + + AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); + + Asset asset1 = new Asset(); + asset1.setName("Test asset 1"); + asset1.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + + JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); + + Asset asset2 = new Asset(); + asset2.setName("Test asset 2"); + asset2.setAssetProfileId(assetProfile.getId()); + + Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); + + JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); + doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(assetProfile.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("z = x + y"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument1 = new Argument(); + ReferencedEntityKey refEntityKey1 = new ReferencedEntityKey("y", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument1.setRefEntityKey(refEntityKey1); + + Argument argument2 = new Argument(); + argument2.setRefEntityId(testDevice.getId()); + ReferencedEntityKey refEntityKey2 = new ReferencedEntityKey("x", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE); + argument2.setRefEntityKey(refEntityKey2); + + config.setArguments(Map.of("x", argument2, "y", argument1)); + + config.setExpression("x + y"); + + Output output = new Output(); + output.setName("z"); + output.setType(OutputType.ATTRIBUTES); + output.setScope(AttributeScope.SERVER_SCOPE); + + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF and perform initial calculation + doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + + // result of asset 2 + ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + + // update device telemetry -> recalculate state for all assets + JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + + Thread.sleep(300); + + // result of asset 1 + ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ1).isNotNull(); + assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + + // result of asset 2 + ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + assertThat(updZ2).isNotNull(); + assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); + +// // update asset 1 telemetry -> recalculate state only for asset 1 +// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 (no changes) +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); +// +// // update asset 2 telemetry -> recalculate state only for asset 2 +// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); +// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); +// +// Thread.sleep(300); +// +// // result of asset 1 (no changes) +// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ1).isNotNull(); +// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); +// +// // result of asset 2 +// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); +// assertThat(updZ2).isNotNull(); +// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 8f56bf491d..9bf3d728aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -46,6 +46,8 @@ public interface CalculatedFieldConfiguration { String getExpression(); + void setExpression(String expression); + Output getOutput(); @JsonIgnore From 96e292fac5cb76511b645553ed83fe7b6b4b53d7 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 15:31:21 +0200 Subject: [PATCH 159/438] Calculated field fixes and improvements --- ...lated-field-arguments-table.component.html | 7 ++-- ...culated-field-arguments-table.component.ts | 20 ++++------ .../calculated-field-dialog.component.html | 2 +- ...ulated-field-argument-panel.component.html | 37 +++++++++++-------- ...ulated-field-argument-panel.component.scss | 22 +++++++++++ ...lculated-field-argument-panel.component.ts | 24 +++++++++++- .../import-export/import-export.service.ts | 1 - .../assets/locale/locale.constant-en_US.json | 1 + 8 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 4b7e516db0..d8a6c7cdb0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -88,9 +88,8 @@ [matTooltip]="'action.edit' | translate" matTooltipPosition="above"> {{ 'calculated-fields.no-arguments' | translate }} }
- @if (errorText && this.argumentsFormArray.dirty) { + @if (errorText) { }
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index a004e3db6f..171c382b7a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -17,9 +17,7 @@ import { ChangeDetectorRef, Component, - effect, forwardRef, - input, Input, OnChanges, Renderer2, @@ -77,8 +75,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces @Input() entityId: EntityId; @Input() tenantId: string; @Input() entityName: string; - - calculatedFieldType = input() + @Input() calculatedFieldType: CalculatedFieldType; errorText = ''; argumentsFormArray = this.fb.array([]); @@ -103,17 +100,12 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces this.argumentsFormArray.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => { this.propagateChange(this.getArgumentsObject()); }); - effect(() => { - if (this.calculatedFieldType() && this.argumentsFormArray.dirty) { - this.argumentsFormArray.updateValueAndValidity(); - } - }); } ngOnChanges(changes: SimpleChanges): void { if (changes.calculatedFieldType?.previousValue && changes.calculatedFieldType.currentValue !== changes.calculatedFieldType.previousValue) { - this.argumentsFormArray.markAsDirty(); + this.argumentsFormArray.updateValueAndValidity(); } } @@ -142,14 +134,16 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { + const argumentObj = this.argumentsFormArray.at(index)?.getRawValue() ?? {}; const ctx = { index, - argument: this.argumentsFormArray.at(index)?.getRawValue() ?? {}, + argument: argumentObj, entityId: this.entityId, - calculatedFieldType: this.calculatedFieldType(), + calculatedFieldType: this.calculatedFieldType, buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, + argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, @@ -171,7 +165,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces } private updateErrorText(): void { - if (this.calculatedFieldType() === CalculatedFieldType.SIMPLE + if (this.calculatedFieldType === CalculatedFieldType.SIMPLE && this.argumentsFormArray.controls.some(control => control.get('refEntityKey').get('type').value === ArgumentType.Rolling)) { this.errorText = 'calculated-fields.hint.arguments-simple-with-rolling'; } else if (!this.argumentsFormArray.controls.length) { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ac60ae8178..1d708f060a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
-
{{ 'calculated-fields.arguments' | translate }}
+
{{ 'calculated-fields.arguments' | translate }}*
warning - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) { + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('duplicateName')) { + + warning + + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('pattern')) {
} @else { -
-
{{ 'calculated-fields.attribute-scope' | translate }}
- - - - {{ 'calculated-fields.server-attributes' | translate }} - - @if (entityType === ArgumentEntityType.Device - || entityType === ArgumentEntityType.Current && entityId.entityType === EntityType.DEVICE) { + @if (isDeviceEntity) { +
+
{{ 'calculated-fields.attribute-scope' | translate }}
+ + + + {{ 'calculated-fields.server-attributes' | translate }} + {{ 'calculated-fields.client-attributes' | translate }} {{ 'calculated-fields.shared-attributes' | translate }} - } - - -
+ + +
+ }
{{ 'calculated-fields.attribute-key' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..45c17628d5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + .time-window-field { + .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { + border-left: 1px solid rgba(0, 0, 0, 0) !important; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 510bdd95f3..1632bd9311 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -52,11 +53,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; + @Input() argumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, this.uniqNameRequired(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -109,6 +111,12 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } + get isDeviceEntity(): boolean { + return this.entityType === ArgumentEntityType.Device + || (this.entityType === ArgumentEntityType.Current + && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) + } + ngOnInit(): void { this.argumentFormGroup.patchValue(this.argument, {emitEvent: false}); this.currentEntityFilter = getCalculatedFieldCurrentEntityFilter(this.entityName, this.entityId); @@ -188,9 +196,21 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); + if (!this.isDeviceEntity) { + this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); + } }); } + private uniqNameRequired(): ValidatorFn { + return (control: UntypedFormControl) => { + const newName = control.value.trim().toLowerCase(); + const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + + return isDuplicate ? { duplicateName: true } : null; + }; + } + private observeEntityKeyChanges(): void { this.argumentFormGroup.get('refEntityKey').get('type').valueChanges .pipe(takeUntilDestroyed()) diff --git a/ui-ngx/src/app/shared/import-export/import-export.service.ts b/ui-ngx/src/app/shared/import-export/import-export.service.ts index 0a49acf8b6..18000bcd4f 100644 --- a/ui-ngx/src/app/shared/import-export/import-export.service.ts +++ b/ui-ngx/src/app/shared/import-export/import-export.service.ts @@ -997,7 +997,6 @@ export class ImportExportService { && !!Object.keys(configuration.arguments).length && isDefined(configuration.expression) && isDefined(configuration.output) - && isNotEmptyStr(configuration.output.name); } private validateImportedImage(image: ImageExportData): boolean { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9bdf533fef..d17409e52d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1064,6 +1064,7 @@ "expression-max-length": "Expression length should be less than 255 characters.", "argument-name-required": "Argument name is required.", "argument-name-pattern": "Argument name is invalid.", + "argument-name-duplicate": "Argument with such name already exists.", "argument-name-max-length": "Argument name should be less than 256 characters.", "argument-type-required": "Argument type is required." } From 8e3c4dc18c3de285f64b111fe62affb03710416a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 7 Feb 2025 15:46:25 +0200 Subject: [PATCH 160/438] Cluster mode fixes --- .../server/actors/ActorSystemContext.java | 6 ++ ...CalculatedFieldEntityMessageProcessor.java | 26 ++++--- ...alculatedFieldManagerMessageProcessor.java | 5 +- .../cf/CalculatedFieldExecutionService.java | 3 - ...efaultCalculatedFieldExecutionService.java | 11 --- .../cf/DefaultCalculatedFieldInitService.java | 9 --- .../server/service/cf/RocksDBService.java | 3 +- ...aultCalculatedFieldEntityProfileCache.java | 1 + .../cf/ctx/CalculatedFieldStateService.java | 4 -- .../KafkaCalculatedFieldStateService.java | 68 +++++++++++++++++++ ...> RocksDBCalculatedFieldStateService.java} | 17 +++-- .../queue/DefaultTbClusterService.java | 10 +-- 12 files changed, 116 insertions(+), 47 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/service/cf/ctx/state/{RocksDBStateService.java => RocksDBCalculatedFieldStateService.java} (92%) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 8f15a6766b..c714ed7dab 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -108,6 +108,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -527,6 +528,11 @@ public class ActorSystemContext { @Getter private CalculatedFieldExecutionService calculatedFieldExecutionService; + @Lazy + @Autowired(required = false) + @Getter + private CalculatedFieldStateService calculatedFieldStateService; + @Lazy @Autowired(required = false) @Getter diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 544dea44c9..0d327adca9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,6 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,6 +68,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; final CalculatedFieldExecutionService cfService; + final CalculatedFieldStateService cfStateService; TbActorCtx ctx; Map states = new HashMap<>(); @@ -76,6 +78,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM this.tenantId = tenantId; this.entityId = entityId; this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); } void init(TbActorCtx ctx) { @@ -102,13 +105,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { MultipleTbCallback multipleTbCallback = new MultipleTbCallback(states.size(), msg.getCallback()); - states.forEach((cfId, state) -> cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); + states.forEach((cfId, state) -> cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), multipleTbCallback)); ctx.stop(ctx.getSelf()); } else { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var state = states.remove(cfId); if (state != null) { - cfService.deleteStateFromStorage(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); + cfStateService.removeState(new CalculatedFieldEntityCtxId(tenantId, cfId, entityId), msg.getCallback()); } } } @@ -178,8 +181,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } - CalculatedFieldState state = getOrInitState(ctx); - if (state.updateState(newArgValues)) { + CalculatedFieldState state = states.get(ctx.getCfId()); + boolean justRestored = false; + if (state == null) { + state = getOrInitState(ctx); + justRestored = true; + } + if (state.updateState(newArgValues) || justRestored) { cfIdList = new ArrayList<>(cfIdList); cfIdList.add(ctx.getCfId()); processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); @@ -222,7 +230,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } else { callback.onSuccess(); // State was updated but no calculation performed; } - cfService.pushStateToStorage(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); + cfStateService.persistState(ctx, new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId), state, callback); } private Map mapToArguments(CalculatedFieldCtx ctx, List data) { diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 6a6edae768..cdc31ed93c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -42,6 +42,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntit import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -68,6 +69,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; @@ -80,6 +82,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); @@ -127,7 +130,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.info("Pushing CF state restore msg to specific actor [{}]", msg.getId().entityId()); getOrCreateActor(msg.getId().entityId()).tell(msg); } else { - cfExecService.deleteStateFromStorage(msg.getId(), msg.getCallback()); + cfStateService.removeState(msg.getId(), msg.getCallback()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 393fbd3ec2..47fd560b4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -43,13 +43,10 @@ public interface CalculatedFieldExecutionService { void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); - void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); - ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculationResult, List cfIds, TbCallback callback); void pushMsgToLinks(CalculatedFieldTelemetryMsg msg, List linkedCalculatedFields, TbCallback callback); - void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 43e85d87e9..e84e785a94 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -141,7 +141,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; @@ -263,16 +262,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - public void pushStateToStorage(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - stateService.persistState(ctx, stateId, state, callback); - } - - @Override - public void deleteStateFromStorage(CalculatedFieldEntityCtxId calculatedFieldEntityCtxId, TbCallback callback) { - stateService.removeState(calculatedFieldEntityCtxId, callback); - } - @Override protected Map>> onAddedPartitions(Set addedPartitions) { var result = new HashMap>>(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index f71617e204..2afd6d8238 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @Slf4j @Service @@ -38,9 +37,6 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { private final CalculatedFieldEntityProfileCache entityProfileCache; - private final CalculatedFieldStateService stateService; - - private final ActorSystemContext actorSystemContext; private final AssetService assetService; private final DeviceService deviceService; @@ -62,9 +58,4 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer } } - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - stateService.restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index fe800f61ad..7181cc43ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -21,6 +21,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -32,7 +33,7 @@ import java.util.Map; @Service @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='monolith'") +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) public class RocksDBService { private final RocksDB db; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 13f95c547d..866cc86f24 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -39,6 +39,7 @@ import java.util.stream.Collectors; @Service @Slf4j @RequiredArgsConstructor +//TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { private static final Integer UNKNOWN = -1; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java index e822d52767..ce1562c735 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -19,12 +19,8 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; -import java.util.Map; - public interface CalculatedFieldStateService { - Map restoreStates(); - void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java new file mode 100644 index 0000000000..9d45532eec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; +import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") +public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { + + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + } + + @Override + public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + callback.onSuccess(); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { + callback.onSuccess(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index b2e33e1705..86556d7adc 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -16,8 +16,10 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -32,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldState import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; @@ -44,12 +47,12 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${service.type:null}'=='monolith'") -public class RocksDBStateService implements CalculatedFieldStateService { +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { + private final ActorSystemContext actorSystemContext; private final RocksDBService rocksDBService; - @Override public Map restoreStates() { return rocksDBService.getAll().entrySet().stream() .collect(Collectors.toMap( @@ -58,6 +61,12 @@ public class RocksDBStateService implements CalculatedFieldStateService { )); } + @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) + public void initCalculatedFieldStates() { + restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); + } + + @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index fa02435ca0..2a090fe701 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -674,7 +674,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } if (deviceNameChanged || deviceProfileChanged) { pushMsgToCore(new DeviceNameOrTypeUpdateMsg(entity.getTenantId(), entity.getId(), entity.getName(), entity.getType()), null); @@ -687,7 +687,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getDeviceProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); sendDeviceStateServiceEvent(entity.getTenantId(), entity.getId(), created, !created, false); @@ -710,7 +710,7 @@ public class DefaultTbClusterService implements TbClusterService { .oldName(old.getName()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } else { ComponentLifecycleMsg msg = ComponentLifecycleMsg.builder() @@ -720,7 +720,7 @@ public class DefaultTbClusterService implements TbClusterService { .profileId(entity.getAssetProfileId()) .name(entity.getName()) .build(); - pushMsgToCalculatedFields(entity.getTenantId(), entity.getId(), ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } broadcastEntityStateChangeEvent(entity.getTenantId(), entity.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } @@ -872,6 +872,6 @@ public class DefaultTbClusterService implements TbClusterService { private void handleCalculatedFieldEntityDeleted(TenantId tenantId, EntityId entityId) { ComponentLifecycleMsg msg = new ComponentLifecycleMsg(tenantId, entityId, ComponentLifecycleEvent.DELETED); - pushMsgToCalculatedFields(tenantId, entityId, ToCalculatedFieldMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); + broadcastToCalculatedFields(ToCalculatedFieldNotificationMsg.newBuilder().setComponentLifecycleMsg(toProto(msg)).build(), TbQueueCallback.EMPTY); } } From 8de861f8977d1da48b85959dd520983aaa6a0876 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 7 Feb 2025 16:11:33 +0200 Subject: [PATCH 161/438] Improve customization for edge-event topic --- application/src/main/resources/thingsboard.yml | 2 ++ .../org/thingsboard/server/queue/discovery/TopicService.java | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4ee08c7095..7b1df1d9d7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1748,6 +1748,8 @@ queue: topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}" # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_EDGE_NOTIFICATIONS_TOPIC:tb_edge.notifications}" + # For edge events messages + event_notifications_topic: "${TB_QUEUE_EDGE_EVENT_NOTIFICATIONS_TOPIC:tb_edge_event.notifications}" # Amount of partitions used by Edge services partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}" # Poll interval for topics related to Edge services diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 254d0a95d6..bab55eece8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -44,6 +44,9 @@ public class TopicService { @Value("${queue.edge.notifications-topic:tb_edge.notifications}") private String tbEdgeNotificationsTopic; + @Value("${queue.edge.event-notifications-topic:tb_edge.notifications}") + private String tbEdgeEventNotificationsTopic; + private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbRuleEngineNotificationTopics = new ConcurrentHashMap<>(); private final ConcurrentMap tbEdgeNotificationTopics = new ConcurrentHashMap<>(); @@ -88,7 +91,7 @@ public class TopicService { } public TopicPartitionInfo buildEdgeEventNotificationsTopicPartitionInfo(TenantId tenantId, EdgeId edgeId) { - return buildTopicPartitionInfo("tb_edge_event.notifications." + tenantId + "." + edgeId, null, null, false); + return buildTopicPartitionInfo(tbEdgeEventNotificationsTopic + "." + tenantId + "." + edgeId, null, null, false); } public String buildTopicName(String topic) { From b54e0cd59c0d70181c509279a80bf985e61fd719 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 7 Feb 2025 16:12:53 +0200 Subject: [PATCH 162/438] Rename tbEdgeEventNotificationsTopic --- .../org/thingsboard/server/queue/discovery/TopicService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index bab55eece8..ca080f9481 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -44,7 +44,7 @@ public class TopicService { @Value("${queue.edge.notifications-topic:tb_edge.notifications}") private String tbEdgeNotificationsTopic; - @Value("${queue.edge.event-notifications-topic:tb_edge.notifications}") + @Value("${queue.edge.event-notifications-topic:tb_edge_event.notifications}") private String tbEdgeEventNotificationsTopic; private final ConcurrentMap tbCoreNotificationTopics = new ConcurrentHashMap<>(); From cfa7066ac62b602d32470ae61ae548604067a15e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:23:38 +0200 Subject: [PATCH 163/438] Resolved comments --- ...culated-field-arguments-table.component.ts | 2 +- ...ulated-field-argument-panel.component.html | 4 ++-- ...ulated-field-argument-panel.component.scss | 22 ------------------- ...lculated-field-argument-panel.component.ts | 13 +++++------ 4 files changed, 9 insertions(+), 32 deletions(-) delete mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts index 171c382b7a..3c51dc7f79 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.ts @@ -143,7 +143,7 @@ export class CalculatedFieldArgumentsTableComponent implements ControlValueAcces buttonTitle: this.argumentsFormArray.at(index)?.value ? 'action.apply' : 'action.add', tenantId: this.tenantId, entityName: this.entityName, - argumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), + usedArgumentNames: this.argumentsFormArray.value.map(({ argumentName }) => argumentName).filter(name => name !== argumentObj.argumentName), }; this.popoverComponent = this.popoverService.displayPopover(trigger, this.renderer, this.viewContainerRef, CalculatedFieldArgumentPanelComponent, 'left', false, null, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index a4f05b3288..4e9819a786 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -111,7 +111,7 @@

} @else { - @if (isDeviceEntity) { + @if (enableAttributeScopeSelection) {
{{ 'calculated-fields.attribute-scope' | translate }}
@@ -155,7 +155,7 @@
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss deleted file mode 100644 index 45c17628d5..0000000000 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -:host ::ng-deep { - .time-window-field { - .mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field.mat-mdc-form-field .mdc-notched-outline__notch { - border-left: 1px solid rgba(0, 0, 0, 0) !important; - } - } -} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 1632bd9311..a2e5545926 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -16,7 +16,7 @@ import { ChangeDetectorRef, Component, Input, OnInit, output } from '@angular/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; -import { FormBuilder, FormGroup, UntypedFormControl, ValidatorFn, Validators } from '@angular/forms'; +import { FormBuilder, FormControl, FormGroup, ValidatorFn, Validators } from '@angular/forms'; import { charsWithNumRegex, noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { ArgumentEntityType, @@ -42,7 +42,6 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', - styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { @@ -53,7 +52,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { @Input() tenantId: string; @Input() entityName: string; @Input() calculatedFieldType: CalculatedFieldType; - @Input() argumentNames: string[]; + @Input() usedArgumentNames: string[]; argumentsDataApplied = output<{ value: CalculatedFieldArgumentValue, index: number }>(); @@ -111,7 +110,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { return this.argumentFormGroup.get('refEntityKey') as FormGroup; } - get isDeviceEntity(): boolean { + get enableAttributeScopeSelection(): boolean { return this.entityType === ArgumentEntityType.Device || (this.entityType === ArgumentEntityType.Current && (this.entityId.entityType === EntityType.DEVICE || this.entityId.entityType === EntityType.DEVICE_PROFILE)) @@ -196,16 +195,16 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit { this.argumentFormGroup.get('refEntityId').get('id').setValue(''); this.argumentFormGroup.get('refEntityId') .get('id')[type === ArgumentEntityType.Tenant || type === ArgumentEntityType.Current ? 'disable' : 'enable'](); - if (!this.isDeviceEntity) { + if (!this.enableAttributeScopeSelection) { this.refEntityKeyFormGroup.get('scope').setValue(AttributeScope.SERVER_SCOPE); } }); } private uniqNameRequired(): ValidatorFn { - return (control: UntypedFormControl) => { + return (control: FormControl) => { const newName = control.value.trim().toLowerCase(); - const isDuplicate = this.argumentNames?.some(name => name.toLowerCase() === newName); + const isDuplicate = this.usedArgumentNames?.some(name => name.toLowerCase() === newName); return isDuplicate ? { duplicateName: true } : null; }; From 0e1c3d66ff9a27cb00cfa3c7e0601a5723282468 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 7 Feb 2025 16:29:27 +0200 Subject: [PATCH 164/438] Added tb-form-panel-title tb-required --- .../dialog/calculated-field-dialog.component.html | 4 ++-- ui-ngx/src/form.scss | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1d708f060a..8d69bf6539 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -65,7 +65,7 @@
-
{{ 'calculated-fields.arguments' | translate }}*
+
{{ 'calculated-fields.arguments' | translate }}
-
{{ 'calculated-fields.expression' | translate }}*
+
{{ 'calculated-fields.expression' | translate }}
@if (fieldFormGroup.get('type').value === CalculatedFieldType.SIMPLE) { diff --git a/ui-ngx/src/form.scss b/ui-ngx/src/form.scss index 0ae2b9584b..4c00a330c7 100644 --- a/ui-ngx/src/form.scss +++ b/ui-ngx/src/form.scss @@ -163,6 +163,13 @@ .tb-form-panel-title { font-weight: 500; font-size: 16px; + + &.tb-required::after { + font-size: 13px; + color: rgba(0, 0, 0, .54); + vertical-align: top; + content: " *"; + } } .tb-form-panel-hint { font-size: 12px; From c6f6408c22f6a724c6d7ed3ec1be68713e76ef05 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 7 Feb 2025 17:04:01 +0200 Subject: [PATCH 165/438] refactored tests --- ...CalculatedFieldEntityMessageProcessor.java | 8 +- .../cf/CalculatedFieldIntegrationTest.java | 159 +++++++++--------- 2 files changed, 79 insertions(+), 88 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 0d327adca9..00913ec66d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index fda0724b1b..f6dc09e3ce 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.cf; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.Test; @@ -35,6 +34,8 @@ import org.thingsboard.server.common.data.cf.configuration.OutputType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.debug.DebugSettings; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; @@ -53,12 +54,8 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes @Test public void testSimpleCalculatedField() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - - JsonNode timeSeries = JacksonUtil.toJsonNode("{\"temperature\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, timeSeries); - - JsonNode attributes = JacksonUtil.toJsonNode("{\"deviceTemperature\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(testDevice.getId()); @@ -72,15 +69,12 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); - config.setArguments(Map.of("T", argument)); - config.setExpression("(T * 9/5) + 32"); Output output = new Output(); output.setName("fahrenheitTemp"); output.setType(OutputType.TIME_SERIES); - config.setOutput(output); calculatedField.setConfiguration(config); @@ -91,19 +85,18 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ObjectNode fahrenheitTemp = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); // update telemetry -> recalculate state - JsonNode newTelemetry = JacksonUtil.toJsonNode("{\"temperature\":30}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, newTelemetry); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); Thread.sleep(300); - ObjectNode fahrenheitTempAfterUpdate = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/timeseries?keys=fahrenheitTemp", ObjectNode.class); - assertThat(fahrenheitTempAfterUpdate).isNotNull(); - assertThat(fahrenheitTempAfterUpdate.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -114,21 +107,20 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureF = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); // update CF argument -> perform calculation with new argument - Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); Thread.sleep(300); - ArrayNode temperatureFAfterUpdateArg = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateArg).isNotNull(); - assertThat(temperatureFAfterUpdateArg.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); @@ -136,36 +128,23 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); - ArrayNode temperatureFAfterUpdateExpression = doGetAsync("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=temperatureF", ArrayNode.class); - assertThat(temperatureFAfterUpdateExpression).isNotNull(); - assertThat(temperatureFAfterUpdateExpression.get(0).get("value").asText()).isEqualTo("104.0"); + temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); - JsonNode deviceAttributes = JacksonUtil.toJsonNode("{\"x\":40}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, deviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":40}")); AssetProfile assetProfile = doPost("/api/assetProfile", createAssetProfile("Test Asset Profile"), AssetProfile.class); - Asset asset1 = new Asset(); - asset1.setName("Test asset 1"); - asset1.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset1 = doPost("/api/asset", asset1, Asset.class); + Asset asset1 = createAsset("Test asset 1", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":11}")); - JsonNode asset1Attributes = JacksonUtil.toJsonNode("{\"y\":11}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset1Attributes); - - Asset asset2 = new Asset(); - asset2.setName("Test asset 2"); - asset2.setAssetProfileId(assetProfile.getId()); - - Asset savedAsset2 = doPost("/api/asset", asset2, Asset.class); - - JsonNode asset2Attributes = JacksonUtil.toJsonNode("{\"y\":12}"); - doPost("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, asset2Attributes); + Asset asset2 = createAsset("Test asset 2", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":12}")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setEntityId(assetProfile.getId()); @@ -205,62 +184,74 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Thread.sleep(300); // result of asset 1 - ArrayNode z1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); assertThat(z1).isNotNull(); assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); // result of asset 2 - ArrayNode z2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); // update device telemetry -> recalculate state for all assets - JsonNode updatedDeviceAttributes = JacksonUtil.toJsonNode("{\"x\":25}"); - doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedDeviceAttributes); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); Thread.sleep(300); // result of asset 1 - ArrayNode updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ1).isNotNull(); - assertThat(updZ1.get(0).get("value").asText()).isEqualTo("36.0"); + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); // result of asset 2 - ArrayNode updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); - assertThat(updZ2).isNotNull(); - assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); - -// // update asset 1 telemetry -> recalculate state only for asset 1 -// JsonNode updatedAsset1Attributes = JacksonUtil.toJsonNode("{\"x\":15}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset1Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 (no changes) -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("37.0"); -// -// // update asset 2 telemetry -> recalculate state only for asset 2 -// JsonNode updatedAsset2Attributes = JacksonUtil.toJsonNode("{\"x\":5}"); -// doPost("/api/plugins/telemetry/DEVICE/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, updatedAsset2Attributes); -// -// Thread.sleep(300); -// -// // result of asset 1 (no changes) -// updZ1 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset1.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ1).isNotNull(); -// assertThat(updZ1.get(0).get("value").asText()).isEqualTo("40.0"); -// -// // result of asset 2 -// updZ2 = doGetAsync("/api/plugins/telemetry/ASSET/" + savedAsset2.getUuidId() + "/values/attributes/SERVER_SCOPE?keys=z", ArrayNode.class); -// assertThat(updZ2).isNotNull(); -// assertThat(updZ2.get(0).get("value").asText()).isEqualTo("30.0"); + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 1 telemetry -> recalculate state only for asset 1 + doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 (no changes) + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + + // update asset 2 telemetry -> recalculate state only for asset 2 + doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); + + Thread.sleep(300); + + // result of asset 1 (no changes) + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + } + + private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/timeseries?keys=" + String.join(",", keys), ObjectNode.class); + } + + private ArrayNode getServerAttributes(EntityId entityId, String... keys) throws Exception { + return doGetAsync("/api/plugins/telemetry/" + entityId.getEntityType() + "/" + entityId.getId() + "/values/attributes/SERVER_SCOPE?keys=" + String.join(",", keys), ArrayNode.class); + } + + private Asset createAsset(String name, AssetProfileId assetProfileId) { + Asset asset = new Asset(); + asset.setName(name); + asset.setAssetProfileId(assetProfileId); + return doPost("/api/asset", asset, Asset.class); } } From 1256993569e75b0cb9c0fb72f04707b9d7900b7a Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 10 Feb 2025 12:59:21 +0200 Subject: [PATCH 166/438] added new tests and fix proto serialization --- .../cf/CalculatedFieldIntegrationTest.java | 200 +++++++++++++++++- .../server/common/util/ProtoUtils.java | 4 +- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index f6dc09e3ce..91aad7b1cd 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -52,7 +52,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes } @Test - public void testSimpleCalculatedField() throws Exception { + public void testSimpleCalculatedFieldWhenAllTelemetryPresent() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"deviceTemperature\":40}")); @@ -69,6 +69,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Argument argument = new Argument(); ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present config.setArguments(Map.of("T", argument)); config.setExpression("(T * 9/5) + 32"); @@ -133,6 +134,99 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); } + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresent() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> state is not ready -> no calculation performed + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> perform calculation + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenNotAllTelemetryPresentButDefaultValueIsSet() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/5) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> perform initial calculation with default value + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + + // update telemetry -> recalculate state + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + } + @Test public void testSimpleCalculatedFieldWhenEntityIdIsProfile() throws Exception { Device testDevice = createDevice("Test device", "1234567890"); @@ -237,6 +331,110 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes z2 = getServerAttributes(asset2.getId(), "z"); assertThat(z2).isNotNull(); assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + + // add new entity to profile -> calculate state for new entity + Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); + doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); + + Thread.sleep(300); + + // result of asset 3 + ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + + // update device telemetry -> recalculate state for all assets + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + + // result of asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + + // update profile for asset 3 -> delete state for asset 3 + AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); + asset3.setAssetProfileId(newAssetProfile.getId()); + asset3 = doPost("/api/asset", asset3, Asset.class); + + // update device telemetry -> recalculate state for asset 1 and asset 2 + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); + + Thread.sleep(300); + + // result of asset 1 + z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + z3 = getServerAttributes(asset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + } + + @Test + public void testSimpleCalculatedFieldWhenExpressionIsInvalid() throws Exception { + Device testDevice = createDevice("Test device", "1234567890"); + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":25}")); + + CalculatedField calculatedField = new CalculatedField(); + calculatedField.setEntityId(testDevice.getId()); + calculatedField.setType(CalculatedFieldType.SIMPLE); + calculatedField.setName("C to F"); + calculatedField.setDebugSettings(DebugSettings.all()); + calculatedField.setConfigurationVersion(1); + + SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); + + Argument argument = new Argument(); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); + argument.setDefaultValue("12"); // not used because real telemetry value in db is present + config.setArguments(Map.of("T", argument)); + config.setExpression("(T * 9/0) + 32"); + + Output output = new Output(); + output.setName("fahrenheitTemp"); + output.setType(OutputType.TIME_SERIES); + config.setOutput(output); + + calculatedField.setConfiguration(config); + calculatedField.setVersion(1L); + + // create CF -> ctx is not initialized -> no calculation perform + CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); + + Thread.sleep(300); + + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + + // update telemetry -> ctx is not initialized -> no calculation perform + doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); + + Thread.sleep(300); + + fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index b81af2bfb9..07406977d2 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -127,8 +127,8 @@ public class ProtoUtils { builder.setProfileIdLSB(msg.getProfileId().getId().getLeastSignificantBits()); } if (msg.getOldProfileId() != null) { - builder.setProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); - builder.setProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); + builder.setOldProfileIdMSB(msg.getOldProfileId().getId().getMostSignificantBits()); + builder.setOldProfileIdLSB(msg.getOldProfileId().getId().getLeastSignificantBits()); } if (msg.getName() != null) { builder.setName(msg.getName()); From 55f279944660e2c2d92825ba799c52719e3d529c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 15:35:14 +0200 Subject: [PATCH 167/438] Implemented Calculated Fields debug test --- .../core/http/calculated-fields.service.ts | 5 + .../calculated-fields-table-config.ts | 34 +++- ...lated-field-arguments-table.component.html | 2 +- ...lculated-field-debug-dialog.component.html | 1 + ...calculated-field-debug-dialog.component.ts | 6 +- .../calculated-field-dialog.component.html | 8 + .../calculated-field-dialog.component.ts | 12 +- .../components/public-api.ts | 1 + ...ulated-field-test-arguments.component.html | 36 ++++ ...lculated-field-test-arguments.component.ts | 88 ++++++++++ ...ed-field-script-test-dialog.component.html | 101 ++++++++++++ ...ed-field-script-test-dialog.component.scss | 51 ++++++ ...ated-field-script-test-dialog.component.ts | 154 ++++++++++++++++++ .../components/event/event-table-config.ts | 10 ++ .../home/components/home-components.module.ts | 10 ++ .../shared/models/calculated-field.models.ts | 12 ++ ui-ngx/src/app/shared/models/entity.models.ts | 2 +- .../assets/locale/locale.constant-en_US.json | 6 + 18 files changed, 527 insertions(+), 12 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 9d8658f124..8e7ab6795e 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,6 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; +import { TestScriptResult } from '@shared/models/rule-node.models'; @Injectable({ providedIn: 'root' @@ -48,4 +49,8 @@ export class CalculatedFieldsService { return this.http.get>(`/api/${entityType}/${id}/calculatedFields${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public testScript(inputParams: any, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 026c249159..8c00cbc925 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,9 +38,13 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData + CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; -import { CalculatedFieldDebugDialogComponent, CalculatedFieldDialogComponent } from './components/public-api'; +import { + CalculatedFieldDebugDialogComponent, + CalculatedFieldDialogComponent, + CalculatedFieldScriptTestDialogComponent +} from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; @@ -53,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id), + action: (id?: CalculatedFieldId, expression?: string) => this.openDebugDialog.call(this, id, expression), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -134,10 +138,10 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id) + action: () => this.openDebugDialog(id, configuration?.expression) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -198,19 +202,22 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, entityId: this.entityId, - id + id, + expression, + testScriptFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -251,4 +258,17 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } + + private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], + data: { + arguments: argumentsObj, + expression, + withApply, + } + }).afterClosed(); + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index d8a6c7cdb0..5ca80b8214 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -24,7 +24,7 @@

{{ 'entity.key' | translate }}
-
+
@for (group of argumentsFormArray.controls; track group) {
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 91da675fea..f88176be8a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -34,6 +34,7 @@ [active]="true" [entityId]="data.id" [functionTestButtonLabel]="'common.test-function' | translate" + (debugEventSelected)="onDebugEventSelected($event)" />
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index d81295948d..5b79476528 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -20,7 +20,7 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; -import { DebugEventType, EventType } from '@shared/models/event.models'; +import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; @@ -51,4 +51,8 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent +
+ +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index d5bf243430..0613ccf1b7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { map, startWith } from 'rxjs/operators'; +import { filter, map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -67,7 +67,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent this.data.additionalDebugActionConfig.action(this.data.value.id) + action: () => this.data.additionalDebugActionConfig.action(this.data.value.id, this.data.value.configuration.expression) } : null; readonly OutputTypeTranslations = OutputTypeTranslations; @@ -110,6 +110,14 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), + this.configFormGroup.get('expressionSCRIPT').value, + true + ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + } + private applyDialogData(): void { const { configuration = {}, type = CalculatedFieldType.SIMPLE, ...value } = this.data.value ?? {}; const { expression, ...restConfig } = configuration as CalculatedFieldConfiguration; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts index 78b8862c2e..14ae73f7e1 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/public-api.ts @@ -18,3 +18,4 @@ export * from './dialog/calculated-field-dialog.component'; export * from './arguments-table/calculated-field-arguments-table.component'; export * from './panel/calculated-field-argument-panel.component'; export * from './debug-dialog/calculated-field-debug-dialog.component'; +export * from './test-dialog/calculated-field-script-test-dialog.component'; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html new file mode 100644 index 0000000000..e6b62a6862 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -0,0 +1,36 @@ + +
+
{{ 'calculated-fields.arguments' | translate }}
+
+
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'common.value' | translate }}
+
+
+ @for (group of argumentsFormArray.controls; track group) { +
+ + + + +
+ } +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts new file mode 100644 index 0000000000..d7a33d1a89 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -0,0 +1,88 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef } from '@angular/core'; +import { + ControlValueAccessor, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + ValidationErrors, + FormBuilder, + FormGroup +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; + +@Component({ + selector: 'tb-calculated-field-test-arguments', + templateUrl: './calculated-field-test-arguments.component.html', + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => CalculatedFieldTestArgumentsComponent), + multi: true, + } + ] +}) +export class CalculatedFieldTestArgumentsComponent extends PageComponent implements ControlValueAccessor, Validator { + + + argumentsFormArray = this.fb.array([]); + + private propagateChange: (value: { argumentName: string; value: unknown }) => void; + + constructor(private fb: FormBuilder) { + super(); + this.argumentsFormArray.valueChanges + .pipe(takeUntilDestroyed()) + .subscribe(() => this.propagateChange(this.getValue())); + } + + registerOnChange(propagateChange: (value: { argumentName: string; value: unknown }) => void): void { + this.propagateChange = propagateChange; + } + + registerOnTouched(_): void { + } + + writeValue(argumentsObj: Record): void { + this.argumentsFormArray.clear(); + Object.keys(argumentsObj).forEach(key => { + this.argumentsFormArray.push(this.fb.group({ + argumentName: [{ value: key, disabled: true}], + value: [argumentsObj[key]] + }) as FormGroup, {emitEvent: false}); + }); + } + + validate(): ValidationErrors | null { + return this.argumentsFormArray.valid ? null : { arguments: { valid: false } }; + } + + private getValue(): { argumentName: string; value: unknown } { + return this.argumentsFormArray.getRawValue().reduce((acc, rowItem) => { + const { argumentName, value } = rowItem; + acc[argumentName] = value; + return acc; + }, {}) as { argumentName: string; value: unknown }; + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html new file mode 100644 index 0000000000..ecfe2d41aa --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -0,0 +1,101 @@ + +
+ +

{{ 'calculated-fields.test-script-function' | translate }} ({{ 'TBEL' }})

+ + +
+
+
+
+
+
+ {{ 'calculated-fields.expression' | translate }} +
+ +
+
+
+
+
+
+ {{ 'calculated-fields.arguments' | translate }} +
+ +
+
+
+
+
+ common.output +
+ +
+
+
+
+
+
+
+ + + + @if (data.withApply) { + + } +
+
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss new file mode 100644 index 0000000000..ee0d59b839 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .test-dialog-container { + .block-label { + padding: 4px; + color: #00acc1; + background: rgba(220, 220, 220, .35); + border-radius: 5px; + } + + .test-block-content { + padding-top: 5px; + padding-left: 5px; + border: 1px solid #c0c0c0; + } + } +} + +:host::ng-deep { + .test-dialog-container { + .gutter { + background-color: #eee; + background-repeat: no-repeat; + background-position: 50%; + } + + .gutter.gutter-horizontal { + cursor: col-resize; + background-image: url("../../../../../../../assets/split.js/grips/horizontal.png"); + } + + .gutter.gutter-vertical { + cursor: row-resize; + background-image: url("../../../../../../../assets/split.js/grips/vertical.png"); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts new file mode 100644 index 0000000000..63ee6fc5a7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -0,0 +1,154 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + DestroyRef, + ElementRef, + Inject, + ViewChild, +} from '@angular/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder } from '@angular/forms'; +import { NEVER, Observable, of, switchMap } from 'rxjs'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { ContentType } from '@shared/models/constants'; +import { JsonContentComponent } from '@shared/components/json-content.component'; +import { ScriptLanguage } from '@shared/models/rule-node.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { beautifyJs } from '@shared/models/beautify.models'; +import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; +import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; + +@Component({ + selector: 'tb-calculated-field-script-test-dialog', + templateUrl: './calculated-field-script-test-dialog.component.html', + styleUrls: ['./calculated-field-script-test-dialog.component.scss'], +}) +export class CalculatedFieldScriptTestDialogComponent extends DialogComponent implements AfterViewInit { + + @ViewChild('leftPanel', {static: true}) leftPanelElmRef: ElementRef; + @ViewChild('rightPanel', {static: true}) rightPanelElmRef: ElementRef; + @ViewChild('topRightPanel', {static: true}) topRightPanelElmRef: ElementRef; + @ViewChild('bottomRightPanel', {static: true}) bottomRightPanelElmRef: ElementRef; + + @ViewChild('expressionContent', {static: true}) expressionContent: JsonContentComponent; + + calculatedFieldScriptTestFormGroup = this.fb.group({ + expression: [], + arguments: [], + output: [] + }); + + readonly ContentType = ContentType; + readonly ScriptLanguage = ScriptLanguage; + readonly functionArgs = Object.keys(this.data.arguments); + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + protected dialogRef: MatDialogRef, + private fb: FormBuilder, + private destroyRef: DestroyRef, + private calculatedFieldService: CalculatedFieldsService) { + super(store, router, dialogRef); + beautifyJs(this.data.expression, {indent_size: 4}).pipe(takeUntilDestroyed()).subscribe( + (res) => { + this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); + } + ); + this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); + } + + ngAfterViewInit(): void { + this.initSplitLayout( + this.leftPanelElmRef.nativeElement, + this.rightPanelElmRef.nativeElement, + this.topRightPanelElmRef.nativeElement, + this.bottomRightPanelElmRef.nativeElement + ); + } + + cancel(): void { + this.dialogRef.close(null); + } + + onTestScript(): void { + this.testScript() + .pipe( + switchMap(output => beautifyJs(output, {indent_size: 4})), + takeUntilDestroyed(this.destroyRef) + ) + .subscribe(output => this.calculatedFieldScriptTestFormGroup.get('output').setValue(output)); + } + + save(): void { + this.testScript().pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { + this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); + this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); + }); + } + + private testScript(): Observable { + if (this.checkInputParamErrors()) { + return this.calculatedFieldService.testScript({ + expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, + arguments: this.calculatedFieldScriptTestFormGroup.get('arguments').value + }).pipe( + switchMap(result => { + if (result.error) { + this.store.dispatch(new ActionNotificationShow( + { + message: result.error, + type: 'error' + })); + return NEVER; + } else { + return of(result.output); + } + }), + ); + } else { + return NEVER; + } + } + + private checkInputParamErrors(): boolean { + this.expressionContent.validateOnSubmit(); + return !this.calculatedFieldScriptTestFormGroup.get('expression').invalid; + } + + private initSplitLayout(leftPanel, rightPanel, topRightPanel, bottomRightPanel): void { + Split([leftPanel, rightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'col-resize' + }); + + Split([topRightPanel, bottomRightPanel], { + sizes: [50, 50], + gutterSize: 8, + cursor: 'row-resize', + direction: 'vertical' + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a937e42bb8..c207e2d161 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -457,6 +457,16 @@ export class EventTableConfig extends EntityTableConfig { }); } break; + case DebugEventType.DEBUG_CALCULATED_FIELD: + this.cellActionDescriptors.push({ + name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), + icon: 'bug_report', + isEnabled: () => true, + onAction: (_, entity) => { + this.debugEventSelected.next(entity.body); + } + }); + break; } this.getTable()?.cellActionDescriptorsUpdated(); } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index b7e35c5406..ea8bf7acf8 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -198,6 +198,12 @@ import { import { CalculatedFieldDebugDialogComponent } from '@home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component'; +import { + CalculatedFieldScriptTestDialogComponent +} from '@home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component'; +import { + CalculatedFieldTestArgumentsComponent +} from '@home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component'; @NgModule({ declarations: @@ -347,6 +353,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], imports: [ CommonModule, @@ -490,6 +498,8 @@ import { CalculatedFieldArgumentsTableComponent, CalculatedFieldArgumentPanelComponent, CalculatedFieldDebugDialogComponent, + CalculatedFieldScriptTestDialogComponent, + CalculatedFieldTestArgumentsComponent, ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index 31dca236a4..e1f3ed8983 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -26,6 +26,7 @@ import { EntityId } from '@shared/models/id/entity-id'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; import { AliasFilterType } from '@shared/models/alias.models'; +import { Observable } from 'rxjs'; export interface CalculatedField extends Omit, 'label'>, HasVersion, HasTenantId, ExportableEntity { debugSettings?: EntityDebugSettings; @@ -126,6 +127,8 @@ export interface CalculatedFieldArgumentValue extends CalculatedFieldArgument { argumentName: string; } +export type CalculatedFieldTestScriptFn = (argumentsObj: Record, expression: string, withApply?: boolean) => Observable; + export interface CalculatedFieldDialogData { value?: CalculatedField; buttonTitle: string; @@ -134,12 +137,21 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig; + testScriptFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldDebugDialogData { id?: CalculatedFieldId; entityId: EntityId; tenantId: string; + expression?: string; + testScriptFn: CalculatedFieldTestScriptFn; +} + +export interface CalculatedFieldScriptTestDialogData { + arguments: Record, + expression: string; + withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index b9d2402850..1f73c0c8eb 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -204,7 +204,7 @@ export interface EntityDebugSettings { } export interface AdditionalDebugActionConfig { - action?: (id?: EntityId) => void; + action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d17409e52d..4822d23387 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -65,6 +65,7 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", + "test": "Test", "done": "Done", "print": "Print", "restore": "Restore", @@ -1018,6 +1019,7 @@ "argument-name": "Argument name", "datasource": "Datasource", "add-argument": "Add argument", + "test-script-function": "Test script function", "no-arguments": "No arguments configured", "argument-settings": "Argument settings", "argument-current": "Current entity", @@ -1107,8 +1109,12 @@ "proceed": "Proceed", "open-details-page": "Open details page", "not-found": "Not found", + "value": "Value", "documentation": "Documentation", "time-left": "{{time}} left", + "output": "Output", + "test-function": "Test function", + "test-with-this-message": "{{test}} with this message", "suffix": { "s": "s", "ms": "ms" From 5efb94dc7ae8f69d1d8e156f42ddb08fb72ca4f0 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 10 Feb 2025 15:58:01 +0200 Subject: [PATCH 168/438] Refactoring --- .../server/actors/ActorSystemContext.java | 6 +- .../server/actors/app/AppActor.java | 1 + .../CalculatedFieldEntityActor.java | 4 + ...CalculatedFieldEntityMessageProcessor.java | 20 +- .../CalculatedFieldManagerActor.java | 5 +- ...alculatedFieldManagerMessageProcessor.java | 15 +- .../server/actors/tenant/TenantActor.java | 1 + ... => CalculatedFieldProcessingService.java} | 16 +- .../cf/CalculatedFieldQueueService.java | 37 +++ .../CalculatedFieldStateService.java | 3 +- .../cf/DefaultCalculatedFieldInitService.java | 2 - ...aultCalculatedFieldProcessingService.java} | 287 +----------------- .../DefaultCalculatedFieldQueueService.java | 238 +++++++++++++++ .../CalculatedFieldEntityProfileCache.java | 1 + ...aultCalculatedFieldEntityProfileCache.java | 9 +- .../KafkaCalculatedFieldStateService.java | 25 +- .../RocksDBCalculatedFieldStateService.java | 2 +- ...faultTbCalculatedFieldConsumerService.java | 22 +- .../queue/DefaultTbClusterService.java | 4 +- .../DefaultTelemetrySubscriptionService.java | 12 +- .../server/common/msg/MsgType.java | 2 + .../cf/CalculatedFieldPartitionChangeMsg.java | 40 +++ .../queue/discovery/HashPartitionService.java | 5 + .../queue/discovery/PartitionService.java | 2 + 24 files changed, 408 insertions(+), 351 deletions(-) rename application/src/main/java/org/thingsboard/server/service/cf/{CalculatedFieldExecutionService.java => CalculatedFieldProcessingService.java} (70%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java rename application/src/main/java/org/thingsboard/server/service/cf/{ctx => }/CalculatedFieldStateService.java (90%) rename application/src/main/java/org/thingsboard/server/service/cf/{DefaultCalculatedFieldExecutionService.java => DefaultCalculatedFieldProcessingService.java} (55%) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index c714ed7dab..c787fc26af 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,9 +106,9 @@ import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -526,7 +526,7 @@ public class ActorSystemContext { @Lazy @Autowired(required = false) @Getter - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; @Lazy @Autowired(required = false) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 8a6907c107..8c70eaf617 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -88,6 +88,7 @@ public class AppActor extends ContextAwareActor { case APP_INIT_MSG: break; case PARTITION_CHANGE_MSG: + case CF_PARTITIONS_CHANGE_MSG: ctx.broadcastToChildren(msg, true); break; case COMPONENT_LIFE_CYCLE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java index cebe6b6a60..5e4aded41f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityActor.java @@ -23,6 +23,7 @@ import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; @Slf4j public class CalculatedFieldEntityActor extends ContextAwareActor { @@ -50,6 +51,9 @@ public class CalculatedFieldEntityActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { + case CF_PARTITIONS_CHANGE_MSG: + processor.process((CalculatedFieldPartitionChangeMsg) msg); + break; case CF_STATE_RESTORE_MSG: processor.process((CalculatedFieldStateRestoreMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 00913ec66d..61f7abb297 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -30,15 +30,16 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -67,8 +68,9 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM final TenantId tenantId; final EntityId entityId; - final CalculatedFieldExecutionService cfService; + final CalculatedFieldProcessingService cfService; final CalculatedFieldStateService cfStateService; + final int partition; TbActorCtx ctx; Map states = new HashMap<>(); @@ -77,14 +79,22 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM super(systemContext); this.tenantId = tenantId; this.entityId = entityId; - this.cfService = systemContext.getCalculatedFieldExecutionService(); + this.cfService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); + this.partition = systemContext.getCalculatedFieldEntityProfileCache().getEntityIdPartition(tenantId, entityId); } void init(TbActorCtx ctx) { this.ctx = ctx; } + public void process(CalculatedFieldPartitionChangeMsg msg) { + if (!msg.getPartitions()[partition]) { + log.info("[{}][{}] Stopping entity actor due to change partition event.", partition, entityId); + ctx.stop(ctx.getSelf()); + } + } + public void process(CalculatedFieldStateRestoreMsg msg) { log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); states.put(msg.getId().cfId(), msg.getState()); @@ -202,7 +212,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldExecutionService().fetchStateFromDb(ctx, entityId); + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 22909dd5af..497602c93b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; /** * Created by ashvayka on 15.03.18. @@ -55,8 +56,8 @@ public class CalculatedFieldManagerActor extends ContextAwareActor { @Override protected boolean doProcess(TbActorMsg msg) { switch (msg.getMsgType()) { - case PARTITION_CHANGE_MSG: - ctx.broadcastToChildren(msg, true); // TODO + case CF_PARTITIONS_CHANGE_MSG: + processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index cdc31ed93c..22492f3c4f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -35,14 +35,15 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -52,6 +53,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,20 +70,20 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final Map> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); - private final CalculatedFieldExecutionService cfExecService; + private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + protected final TenantId tenantId; protected TbActorCtx ctx; - final TenantId tenantId; CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); - this.cfExecService = systemContext.getCalculatedFieldExecutionService(); + this.cfExecService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); this.assetProfileCache = systemContext.getAssetProfileCache(); @@ -478,4 +480,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware oldLinks.forEach(link -> entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).remove(link)); } + public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + ctx.broadcastToChildren(msg, true); + } } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 1b008f462a..16ea2c1398 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -170,6 +170,7 @@ public class TenantActor extends RuleChainManagerActor { case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: + case CF_PARTITIONS_CHANGE_MSG: case CF_ENTITY_LIFECYCLE_MSG: onToCalculatedFieldSystemActorMsg((ToCalculatedFieldSystemMsg) msg, true); break; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java similarity index 70% rename from application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java index 47fd560b4d..24b428593a 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldProcessingService.java @@ -15,15 +15,11 @@ */ package org.thingsboard.server.service.cf; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; @@ -31,17 +27,7 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.List; -public interface CalculatedFieldExecutionService { - - /** - * Filter CFs based on the request entity. Push to the queue if any matching CF exist; - * - * @param request - telemetry save request; - * @param callback - */ - void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); - - void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); +public interface CalculatedFieldProcessingService { ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java new file mode 100644 index 0000000000..b84b54af81 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldQueueService.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; + +import java.util.List; + +public interface CalculatedFieldQueueService { + + /** + * Filter CFs based on the request entity. Push to the queue if any matching CF exist; + * + * @param request - telemetry save request; + * @param callback + */ + void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback); + + void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java rename to application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index ce1562c735..37211c66c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.cf.ctx; +package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 2afd6d8238..f8eb7e852e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -20,8 +20,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java similarity index 55% rename from application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java rename to application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index e84e785a94..6b47053f2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; @@ -32,25 +31,17 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.rule.engine.api.AttributesSaveRequest; -import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; import org.thingsboard.server.actors.calculatedField.MultipleTbCallback; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.OutputType; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -59,7 +50,6 @@ import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; @@ -67,12 +57,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; @@ -80,12 +67,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinke import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -93,149 +79,51 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; -import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; -import org.thingsboard.server.service.partition.AbstractPartitionBasedService; -import org.thingsboard.server.service.profile.TbAssetProfileCache; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; -import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; -import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; -import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; +@TbRuleEngineComponent @Service @Slf4j @RequiredArgsConstructor -public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService implements CalculatedFieldExecutionService { +public class DefaultCalculatedFieldProcessingService implements CalculatedFieldProcessingService { - public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - } - - @Override - public void onFailure(Throwable t) { - } - }; - - private final TbAssetProfileCache assetProfileCache; - private final TbDeviceProfileCache deviceProfileCache; - private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; private final TbClusterService clusterService; private final ApiLimitService apiLimitService; + private final PartitionService partitionService; - private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor; - private final ConcurrentMap states = new ConcurrentHashMap<>(); - - private static final Set supportedReferencedEntities = EnumSet.of( - EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT - ); - @Value("${calculatedField.initFetchPackSize:50000}") @Getter private int initFetchPackSize; @PostConstruct public void init() { - super.init(); - calculatedFieldExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( - Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); } @PreDestroy public void stop() { - super.stop(); - if (calculatedFieldExecutor != null) { - calculatedFieldExecutor.shutdownNow(); - } if (calculatedFieldCallbackExecutor != null) { calculatedFieldCallbackExecutor.shutdownNow(); } } - @Override - protected String getServiceName() { - return "Calculated Field Execution"; - } - - @Override - protected String getSchedulerExecutorName() { - return "calculated-field-scheduled"; - } - - @Override - public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; - //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; - //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - @Override - public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { - var tenantId = request.getTenantId(); - var entityId = request.getEntityId(); - checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), - () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); - } - - private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, - Predicate mainEntityFilter, Predicate linkedEntityFilter, - Supplier msg, FutureCallback callback) { - boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); - if (send) { - clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); - } else { - if (callback != null) { - callback.onSuccess(null); - } - } - } - - private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { - boolean send = false; - if (supportedReferencedEntities.contains(entityId.getEntityType())) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); - if (!send) { - send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); - } - if (!send) { - send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() - .map(CalculatedFieldLink::getCalculatedFieldId) - .map(calculatedFieldCache::getCalculatedFieldCtx) - .anyMatch(linkedEntityFilter); - } - } - return send; - } - @Override public ListenableFuture fetchStateFromDb(CalculatedFieldCtx ctx, EntityId entityId) { Map> argFutures = new HashMap<>(); @@ -262,21 +150,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }, calculatedFieldCallbackExecutor); } - @Override - protected Map>> onAddedPartitions(Set addedPartitions) { - var result = new HashMap>>(); - return result; - } - - @Override - protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { - cleanupEntity(entityId); - } - - private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - } - @Override public void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, CalculatedFieldResult calculatedFieldResult, List cfIds, TbCallback callback) { try { @@ -369,30 +242,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { - Map argumentValues = new HashMap<>(); - List> futures = new ArrayList<>(); - necessaryArguments.forEach((key, argument) -> { - futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument), - result -> { - argumentValues.put(key, result); - return result; - }, calculatedFieldCallbackExecutor)); - }); - return Futures.transform(Futures.allAsList(futures), results -> { - onComplete.accept(argumentValues); - return null; - }, calculatedFieldCallbackExecutor); - } - - private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) - ? targetEntityId - : argumentEntityId; - return fetchKvEntry(tenantId, entityId, argument); - } - private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); @@ -462,78 +311,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas }; } - private boolean isProfileEntity(EntityId entityId) { - return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType()); - } - - private EntityId getProfileId(TenantId tenantId, EntityId entityId) { - return switch (entityId.getEntityType()) { - case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); - case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> null; - }; - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - List entries = request.getEntries(); - List versions = result.getVersions(); - for (int i = 0; i < entries.size(); i++) { - long tsVersion = versions.get(i); - TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); - telemetryMsg.addTsData(tsProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { - ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); - - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); - telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); - List entries = request.getEntries(); - for (int i = 0; i < entries.size(); i++) { - long attrVersion = versions.get(i); - AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); - telemetryMsg.addAttrData(attrProto); - } - msg.setTelemetryMsg(telemetryMsg.build()); - - return msg.build(); - } - - private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { - CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); - - telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); - telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); - - telemetryMsg.setEntityType(entityId.getEntityType().name()); - telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); - telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - - if (calculatedFieldIds != null) { - for (CalculatedFieldId cfId : calculatedFieldIds) { - telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); - } - } - - if (tbMsgId != null) { - telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); - telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); - } - - if (tbMsgType != null) { - telemetryMsg.setTbMsgType(tbMsgType.name()); - } - - return telemetryMsg; - } - private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { return CalculatedFieldIdProto.newBuilder() .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) @@ -541,60 +318,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas .build(); } - private CalculatedFieldTelemetryUpdateRequest fromProto(CalculatedFieldTelemetryMsgProto proto) { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - - if (!proto.getTsDataList().isEmpty()) { - List updatedTelemetry = proto.getTsDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } else { - AttributeScope scope = AttributeScope.valueOf(proto.getScope().name()); - List updatedTelemetry = proto.getAttrDataList().stream() - .map(ProtoUtils::fromProto) - .toList(); - return new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, scope, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); - } - } - - private static TbQueueCallback wrap(FutureCallback callback) { - if (callback != null) { - return new FutureCallbackWrapper(callback); - } else { - return DUMMY_TB_QUEUE_CALLBACK; - } - } - - private static class FutureCallbackWrapper implements TbQueueCallback { - private final FutureCallback callback; - - public FutureCallbackWrapper(FutureCallback callback) { - this.callback = callback; - } - - @Override - public void onSuccess(TbQueueMsgMetadata metadata) { - callback.onSuccess(null); - } - - @Override - public void onFailure(Throwable t) { - callback.onFailure(t); - } - } - private static class TbCallbackWrapper implements TbQueueCallback { private final TbCallback callback; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java new file mode 100644 index 0000000000..9d1f9b6db5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldQueueService.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import com.google.common.util.concurrent.FutureCallback; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.TimeseriesSaveResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeScopeProto; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; +import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.profile.TbAssetProfileCache; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.function.Predicate; +import java.util.function.Supplier; + +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; + +@Service +@Slf4j +@RequiredArgsConstructor +public class DefaultCalculatedFieldQueueService implements CalculatedFieldQueueService { + + public static final TbQueueCallback DUMMY_TB_QUEUE_CALLBACK = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + } + }; + + private final TbAssetProfileCache assetProfileCache; + private final TbDeviceProfileCache deviceProfileCache; + private final CalculatedFieldCache calculatedFieldCache; + private final TbClusterService clusterService; + + private static final Set supportedReferencedEntities = EnumSet.of( + EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT + ); + + @Value("${calculatedField.initFetchPackSize:50000}") + @Getter + private int initFetchPackSize; + + @Override + public void pushRequestToQueue(TimeseriesSaveRequest request, TimeseriesSaveResult result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + //TODO: 1. check that request entity has calculated fields for entity or profile. If yes - push to corresponding partitions; + //TODO: 2. check that request entity has calculated field links. If yes - push to corresponding partitions; + //TODO: in 1 and 2 we should do the check as quick as possible. Should we also check the field/link keys?; + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries()), cf -> cf.linkMatches(entityId, request.getEntries()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + @Override + public void pushRequestToQueue(AttributesSaveRequest request, List result, FutureCallback callback) { + var tenantId = request.getTenantId(); + var entityId = request.getEntityId(); + checkEntityAndPushToQueue(tenantId, entityId, cf -> cf.matches(request.getEntries(), request.getScope()), cf -> cf.linkMatches(entityId, request.getEntries(), request.getScope()), + () -> toCalculatedFieldTelemetryMsgProto(request, result), callback); + } + + private void checkEntityAndPushToQueue(TenantId tenantId, EntityId entityId, + Predicate mainEntityFilter, Predicate linkedEntityFilter, + Supplier msg, FutureCallback callback) { + boolean send = checkEntityForCalculatedFields(tenantId, entityId, mainEntityFilter, linkedEntityFilter); + if (send) { + clusterService.pushMsgToCalculatedFields(tenantId, entityId, msg.get(), wrap(callback)); + } else { + if (callback != null) { + callback.onSuccess(null); + } + } + } + + private boolean checkEntityForCalculatedFields(TenantId tenantId, EntityId entityId, Predicate filter, Predicate linkedEntityFilter) { + boolean send = false; + if (supportedReferencedEntities.contains(entityId.getEntityType())) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(entityId).stream().anyMatch(filter); + if (!send) { + send = calculatedFieldCache.getCalculatedFieldCtxsByEntityId(getProfileId(tenantId, entityId)).stream().anyMatch(filter); + } + if (!send) { + send = calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId).stream() + .map(CalculatedFieldLink::getCalculatedFieldId) + .map(calculatedFieldCache::getCalculatedFieldCtx) + .anyMatch(linkedEntityFilter); + } + } + return send; + } + + private EntityId getProfileId(TenantId tenantId, EntityId entityId) { + return switch (entityId.getEntityType()) { + case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); + case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); + default -> null; + }; + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(TimeseriesSaveRequest request, TimeseriesSaveResult result) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + List entries = request.getEntries(); + List versions = result.getVersions(); + for (int i = 0; i < entries.size(); i++) { + long tsVersion = versions.get(i); + TsKvProto tsProto = toTsKvProto(entries.get(i)).toBuilder().setVersion(tsVersion).build(); + telemetryMsg.addTsData(tsProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private ToCalculatedFieldMsg toCalculatedFieldTelemetryMsgProto(AttributesSaveRequest request, List versions) { + ToCalculatedFieldMsg.Builder msg = ToCalculatedFieldMsg.newBuilder(); + + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = buildTelemetryMsgProto(request.getTenantId(), request.getEntityId(), request.getPreviousCalculatedFieldIds(), request.getTbMsgId(), request.getTbMsgType()); + telemetryMsg.setScope(AttributeScopeProto.valueOf(request.getScope().name())); + List entries = request.getEntries(); + for (int i = 0; i < entries.size(); i++) { + long attrVersion = versions.get(i); + AttributeValueProto attrProto = ProtoUtils.toProto(entries.get(i)).toBuilder().setVersion(attrVersion).build(); + telemetryMsg.addAttrData(attrProto); + } + msg.setTelemetryMsg(telemetryMsg.build()); + + return msg.build(); + } + + private CalculatedFieldTelemetryMsgProto.Builder buildTelemetryMsgProto(TenantId tenantId, EntityId entityId, List calculatedFieldIds, UUID tbMsgId, TbMsgType tbMsgType) { + CalculatedFieldTelemetryMsgProto.Builder telemetryMsg = CalculatedFieldTelemetryMsgProto.newBuilder(); + + telemetryMsg.setTenantIdMSB(tenantId.getId().getMostSignificantBits()); + telemetryMsg.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()); + + telemetryMsg.setEntityType(entityId.getEntityType().name()); + telemetryMsg.setEntityIdMSB(entityId.getId().getMostSignificantBits()); + telemetryMsg.setEntityIdLSB(entityId.getId().getLeastSignificantBits()); + + if (calculatedFieldIds != null) { + for (CalculatedFieldId cfId : calculatedFieldIds) { + telemetryMsg.addPreviousCalculatedFields(toProto(cfId)); + } + } + + if (tbMsgId != null) { + telemetryMsg.setTbMsgIdMSB(tbMsgId.getMostSignificantBits()); + telemetryMsg.setTbMsgIdLSB(tbMsgId.getLeastSignificantBits()); + } + + if (tbMsgType != null) { + telemetryMsg.setTbMsgType(tbMsgType.name()); + } + + return telemetryMsg; + } + + private CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + + private static TbQueueCallback wrap(FutureCallback callback) { + if (callback != null) { + return new FutureCallbackWrapper(callback); + } else { + return DUMMY_TB_QUEUE_CALLBACK; + } + } + + private static class FutureCallbackWrapper implements TbQueueCallback { + private final FutureCallback callback; + + public FutureCallbackWrapper(FutureCallback callback) { + this.callback = callback; + } + + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + callback.onSuccess(null); + } + + @Override + public void onFailure(Throwable t) { + callback.onFailure(t); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java index 6b0718eb84..fec8c8ba12 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -32,4 +32,5 @@ public interface CalculatedFieldEntityProfileCache extends ApplicationListener

getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + int getEntityIdPartition(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 866cc86f24..9a56d35ac4 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -42,7 +42,7 @@ import java.util.stream.Collectors; //TODO: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { - private static final Integer UNKNOWN = -1; + private static final Integer UNKNOWN = 0; private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); private final PartitionService partitionService; private volatile List myPartitions = Collections.emptyList(); @@ -84,4 +84,11 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); } + + @Override + public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { + var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + return tpi.getPartition().orElse(UNKNOWN); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 9d45532eec..a1999d438c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -17,34 +17,11 @@ package org.thingsboard.server.service.cf.ctx.state; import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; - -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; @Service @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 86556d7adc..48e17954ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -37,7 +37,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.cf.RocksDBService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import java.util.Map; import java.util.Optional; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 3d8bbddc80..f3e976084b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,6 +21,7 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -32,8 +33,10 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -49,7 +52,6 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -58,6 +60,7 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -134,8 +137,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - log.debug("Subscribing to partitions: {}", event.getCalculatedFieldsPartitions()); + var partitions = event.getCalculatedFieldsPartitions(); + log.info("Subscribing to partitions: {}", partitions); + // TODO: @vklimov - before update of the main consumer, we should read the state topics and use + // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. + // Cleanup entities that do not belong to current partition; mainConsumer.update(event.getCalculatedFieldsPartitions()); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } + + private boolean[] partitionsToBooleanIndexArray(Set partitions) { + boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; + for(var tpi : partitions) { + tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); + } + return myPartitions; } private void processMsgs(List> msgs, TbQueueConsumer> consumer, CalculatedFieldQueueConfig config) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2a090fe701..48d26895b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -96,7 +96,7 @@ import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -148,7 +148,7 @@ public class DefaultTbClusterService implements TbClusterService { @Autowired @Lazy - private CalculatedFieldExecutionService calculatedFieldExecutionService; + private CalculatedFieldProcessingService calculatedFieldProcessingService; private final TopicService topicService; private final TbDeviceProfileCache deviceProfileCache; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 3f7f9c0b7d..b338a8382c 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -50,7 +50,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; -import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; +import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; @@ -77,7 +77,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final TbEntityViewService tbEntityViewService; private final TbApiUsageReportClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; - private final CalculatedFieldExecutionService calculatedFieldExecutionService; + private final CalculatedFieldQueueService calculatedFieldQueueService; private ExecutorService tsCallBackExecutor; @@ -89,13 +89,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer @Lazy TbEntityViewService tbEntityViewService, TbApiUsageReportClient apiUsageClient, TbApiUsageStateService apiUsageStateService, - CalculatedFieldExecutionService calculatedFieldExecutionService) { + CalculatedFieldQueueService calculatedFieldQueueService) { this.attrService = attrService; this.tsService = tsService; this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; - this.calculatedFieldExecutionService = calculatedFieldExecutionService; + this.calculatedFieldQueueService = calculatedFieldQueueService; } @PostConstruct @@ -147,7 +147,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer resultFuture = tsService.saveWithoutLatest(tenantId, entityId, request.getEntries(), request.getTtl()); } DonAsynchron.withCallback(resultFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(resultFuture, success -> onTimeSeriesUpdate(tenantId, entityId, request.getEntries())); if (request.isSaveLatest() && !request.isOnlyLatest()) { @@ -167,7 +167,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer log.trace("Executing saveInternal [{}]", request); ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); DonAsynchron.withCallback(saveFuture, result -> { - calculatedFieldExecutionService.pushRequestToQueue(request, result, request.getCallback()); + calculatedFieldQueueService.pushRequestToQueue(request, result, request.getCallback()); }, safeCallback(request.getCallback()), tsCallBackExecutor); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index e13e2d2ecb..9e5f00abf2 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -139,6 +139,8 @@ public enum MsgType { CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; + CF_PARTITIONS_CHANGE_MSG, // Sent when cluster event occures; + CF_ENTITY_LIFECYCLE_MSG, // Sent on CF/Device/Asset create/update/delete; CF_TELEMETRY_MSG, // Sent from queue to actor system; CF_LINKED_TELEMETRY_MSG, // Sent from queue to actor system; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java new file mode 100644 index 0000000000..5ab4b0ad9f --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldPartitionChangeMsg.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.cf.CalculatedField; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +import java.util.Set; + +@Data +public class CalculatedFieldPartitionChangeMsg implements ToCalculatedFieldSystemMsg { + + private final boolean[] partitions; + + @Override + public TenantId getTenantId() { + return TenantId.SYS_TENANT_ID; + } + + @Override + public MsgType getMsgType() { + return MsgType.CF_PARTITIONS_CHANGE_MSG; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 5c6bf545b5..f2f1ccd19c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -526,6 +526,11 @@ public class HashPartitionService implements PartitionService { return list == null ? 0 : list.size(); } + @Override + public int getTotalCalculatedFieldPartitions() { + return cfPartitions; + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 8e2152830e..b0e1229f97 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -77,4 +77,6 @@ public interface PartitionService { boolean isManagedByCurrentService(TenantId tenantId); + int getTotalCalculatedFieldPartitions(); + } From 84b9bde5772020787e4e842d1fd5d6971e35fc7d Mon Sep 17 00:00:00 2001 From: mpetrov Date: Mon, 10 Feb 2025 16:11:46 +0200 Subject: [PATCH 169/438] Refactoring --- ui-ngx/src/app/core/http/calculated-fields.service.ts | 6 +++--- ui-ngx/src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields/calculated-fields-table-config.ts | 3 ++- .../calculated-field-debug-dialog.component.scss | 1 - .../dialog/calculated-field-dialog.component.ts | 5 ++++- .../calculated-field-script-test-dialog.component.html | 4 ++-- .../calculated-field-script-test-dialog.component.ts | 7 +++---- .../modules/home/components/event/event-table-config.ts | 8 +++----- ui-ngx/src/app/shared/models/calculated-field.models.ts | 7 +++++-- ui-ngx/src/app/shared/models/entity.models.ts | 5 +++++ ui-ngx/src/app/shared/models/rule-node.models.ts | 5 ----- 11 files changed, 28 insertions(+), 25 deletions(-) diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index 8e7ab6795e..acaf3b2817 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -19,10 +19,10 @@ import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { PageData } from '@shared/models/page/page-data'; -import { CalculatedField } from '@shared/models/calculated-field.models'; +import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/rule-node.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: any, config?: RequestConfig): Observable { + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index e3353989cc..c1333df1ac 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,7 +35,6 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, - TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -44,6 +43,7 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; +import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 8c00cbc925..c71eef8de3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -38,7 +38,8 @@ import { catchError, filter, switchMap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, - CalculatedFieldDialogData, CalculatedFieldScriptTestDialogData + CalculatedFieldDialogData, + CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss index 23aa4070d7..19bf072b11 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.scss @@ -16,7 +16,6 @@ :host { .debug-dialog-container { height: 77vh; - min-width: 80vw; .debug-dialog-content { border-radius: 0; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 0613ccf1b7..20a37c8cc3 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -115,7 +115,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent [k, ''])), this.configFormGroup.get('expressionSCRIPT').value, true - ).pipe(filter(Boolean)).subscribe((expression: string) => this.configFormGroup.get('expressionSCRIPT').setValue(expression)); + ).pipe(filter(Boolean)).subscribe((expression: string) => { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index ecfe2d41aa..7af5e55b1c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -48,10 +48,10 @@

-
+
{{ 'calculated-fields.arguments' | translate }}
- +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index 63ee6fc5a7..fa66827eb6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,6 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; +import { filter } from 'rxjs/operators'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -71,10 +72,8 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { - this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}); - } + beautifyJs(this.data.expression, {indent_size: 4}).pipe(filter(Boolean), takeUntilDestroyed()).subscribe( + (res) => this.calculatedFieldScriptTestFormGroup.get('expression').patchValue(res, {emitEvent: false}) ); this.calculatedFieldScriptTestFormGroup.get('arguments').patchValue(this.data.arguments, {emitEvent: false}); } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index c207e2d161..39fb143de0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 6)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 6)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -462,9 +462,7 @@ export class EventTableConfig extends EntityTableConfig { name: this.translate.instant('common.test-with-this-message', {test: this.translate.instant(this.testButtonLabel)}), icon: 'bug_report', isEnabled: () => true, - onAction: (_, entity) => { - this.debugEventSelected.next(entity.body); - } + onAction: (_, entity) => this.debugEventSelected.next(entity.body) }); break; } diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index e1f3ed8983..1a208e122f 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -148,10 +148,13 @@ export interface CalculatedFieldDebugDialogData { testScriptFn: CalculatedFieldTestScriptFn; } -export interface CalculatedFieldScriptTestDialogData { +export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { + withApply: boolean; +} + +export interface CalculatedFieldTestScriptInputParams { arguments: Record, expression: string; - withApply: boolean; } export interface ArgumentEntityTypeParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 1f73c0c8eb..3a8c06f544 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,6 +203,11 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } +export interface TestScriptResult { + output: string; + error: string; +} + export interface AdditionalDebugActionConfig { action?: (id?: EntityId, ...restArguments: unknown[]) => void; title: string; diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index a79fdbc376..ac08fa38ac 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -374,11 +374,6 @@ export interface TestScriptInputParams { msgType: string; } -export interface TestScriptResult { - output: string; - error: string; -} - export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', From 5563da7c0d9e48fc707626e23db0082c6bf68faa Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 10 Feb 2025 16:28:59 +0200 Subject: [PATCH 170/438] implemented filters for AlarmCountQuery --- ...efaultTbEntityDataSubscriptionService.java | 8 +- .../subscription/TbAlarmCountSubCtx.java | 37 +++++- .../server/controller/WebsocketApiTest.java | 114 +++++++++++++++++- .../server/dao/alarm/AlarmService.java | 2 + .../common/data/query/AlarmCountQuery.java | 7 ++ .../common/data/query/EntityCountQuery.java | 2 +- .../server/dao/alarm/AlarmDao.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 8 +- .../server/dao/sql/alarm/JpaAlarmDao.java | 4 +- .../dao/sql/query/AlarmQueryRepository.java | 2 +- .../query/DefaultAlarmQueryRepository.java | 11 +- .../server/dao/service/AlarmServiceTest.java | 51 ++++++++ 12 files changed, 230 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 946a425a71..9812872bf1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -33,7 +33,6 @@ import org.springframework.stereotype.Service; import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -41,7 +40,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.data.query.ComparisonTsValue; -import org.thingsboard.server.common.data.query.OriginatorAlarmFilter; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityKey; @@ -55,7 +53,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AggHistoryCmd; @@ -65,7 +62,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusCmd; -import org.thingsboard.server.service.ws.telemetry.cmd.v2.CmdUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; @@ -74,7 +70,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.GetTsCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.UnsubscribeCmd; -import org.thingsboard.server.service.ws.telemetry.sub.AlarmSubscriptionUpdate; import java.util.ArrayList; import java.util.Arrays; @@ -83,7 +78,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; @@ -555,7 +549,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { Map sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, - attributesService, stats, alarmService, sessionRef, cmd.getCmdId()); + attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription); if (cmd.getQuery() != null) { ctx.setAndResolveQuery(cmd.getQuery()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index af7cfcfcfe..8425b2a57a 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -19,20 +19,33 @@ import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; +import java.util.List; + @Slf4j @ToString(callSuper = true) public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx { private final AlarmService alarmService; + private final int maxEntitiesPerAlarmSubscription; + @Getter @Setter private volatile int result; @@ -40,20 +53,21 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + List entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); + if (entityIds.isEmpty()) { + return 0; + } else { + return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); + } + + } + + private EntityDataQuery buildEntityDataQuery() { + EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, + new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); + return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, null); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index b64c87ccbd..e72e1dc806 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -83,7 +83,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j @DaoSqlTest @TestPropertySource(properties = { - "server.ws.alarms_per_alarm_status_subscription_cache_size=5" + "server.ws.alarms_per_alarm_status_subscription_cache_size=5", + "server.ws.dynamic_page_link.refresh_interval=2" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -324,6 +325,117 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertEquals(1, update.getCount()); } + @Test + public void testAlarmCountWsCmdWithSingleEntityFilter() throws Exception { + loginTenantAdmin(); + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(tenantId); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(singleEntityFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + Alarm alarm = new Alarm(); + alarm.setOriginator(tenantId); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + + alarm = doPost("/api/alarm", alarm, Alarm.class); + + AlarmCountCmd cmd2 = new AlarmCountCmd(2, alarmCountQuery); + + getWsClient().send(cmd2); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(2, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + singleEntityFilter.setSingleEntity(tenantAdminUserId); + AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + alarm.setAssigneeId(tenantAdminUserId); + alarm = doPost("/api/alarm", alarm, Alarm.class); + + singleEntityFilter.setSingleEntity(tenantId); + alarmCountQuery.setAssigneeId(tenantAdminUserId); + AlarmCountCmd cmd4 = new AlarmCountCmd(4, alarmCountQuery); + + getWsClient().send(cmd4); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(4, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + alarmCountQuery.setSeverityList(Collections.singletonList(AlarmSeverity.CRITICAL)); + AlarmCountCmd cmd5 = new AlarmCountCmd(5, alarmCountQuery); + + getWsClient().send(cmd5); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(5, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + alarm.setSeverity(AlarmSeverity.CRITICAL); + doPost("/api/alarm", alarm, Alarm.class); + + AlarmCountCmd cmd6 = new AlarmCountCmd(6, alarmCountQuery); + + getWsClient().send(cmd6); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(6, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + } + + @Test + public void testAlarmCountWsCmdWithDeviceType() throws Exception { + loginTenantAdmin(); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceTypes(List.of("default")); + AlarmCountQuery alarmCountQuery = new AlarmCountQuery(deviceTypeFilter); + AlarmCountCmd cmd1 = new AlarmCountCmd(1, alarmCountQuery); + + getWsClient().send(cmd1); + + AlarmCountUpdate update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + + getWsClient().registerWaitForUpdate(); + + Alarm alarm = new Alarm(); + alarm.setOriginator(device.getId()); + alarm.setType("TEST ALARM"); + alarm.setSeverity(AlarmSeverity.WARNING); + + alarm = doPost("/api/alarm", alarm, Alarm.class); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(3000)); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(1, update.getCount()); + + deviceTypeFilter.setDeviceTypes(List.of("non-existing")); + AlarmCountCmd cmd3 = new AlarmCountCmd(3, alarmCountQuery); + + getWsClient().send(cmd3); + + update = getWsClient().parseAlarmCountReply(getWsClient().waitForReply()); + Assert.assertEquals(3, update.getCmdId()); + Assert.assertEquals(0, update.getCount()); + } + @Test public void testAlarmStatusWsCmd() throws Exception { loginTenantAdmin(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index ce040595cc..11252b59a5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -118,6 +118,8 @@ public interface AlarmService extends EntityDaoService { long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); + PageData findAlarmTypesByTenantId(TenantId tenantId, PageLink pageLink); List findActiveOriginatorAlarms(TenantId tenantId, OriginatorAlarmFilter originatorAlarmFilter, int limit); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java index 19cb445380..bb969a28ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; +import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -30,6 +31,7 @@ import java.util.List; @NoArgsConstructor @AllArgsConstructor @Getter +@Setter @ToString public class AlarmCountQuery extends EntityCountQuery { private long startTs; @@ -40,4 +42,9 @@ public class AlarmCountQuery extends EntityCountQuery { private List severityList; private boolean searchPropagatedAlarms; private UserId assigneeId; + + public AlarmCountQuery(EntityFilter entityFilter) { + super(entityFilter); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index 3dd24147e7..be9fa4c664 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -27,7 +27,7 @@ import java.util.List; public class EntityCountQuery { @Getter - private EntityFilter entityFilter; + protected EntityFilter entityFilter; @Getter protected List keyFilters; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 4b4ee0ae4f..2432bb4bbe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -106,7 +106,7 @@ public interface AlarmDao extends Dao { AlarmApiCallResult unassignAlarm(TenantId tenantId, AlarmId alarmId, long unassignTime); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); PageData findTenantAlarmTypes(UUID tenantId, PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index bf08621d14..14cd65185d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -352,7 +352,13 @@ public class BaseAlarmService extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - return alarmDao.countAlarmsByQuery(tenantId, customerId, query); + return alarmDao.countAlarmsByQuery(tenantId, customerId, query, Collections.emptyList()); + } + + @Override + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + return alarmDao.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index e433dfbd86..a3e02d5461 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -415,8 +415,8 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { - return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query); + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { + return alarmQueryRepository.countAlarmsByQuery(tenantId, customerId, query, orderedEntityIds); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java index 49aa6c7276..2a26d62376 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/AlarmQueryRepository.java @@ -30,6 +30,6 @@ public interface AlarmQueryRepository { PageData findAlarmDataByQueryForEntities(TenantId tenantId, AlarmDataQuery query, Collection orderedEntityIds); - long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query); + long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 05da22f9a5..b4cc40fc6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.stream.Collectors; @Repository @@ -314,7 +315,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } @Override - public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query) { + public long countAlarmsByQuery(TenantId tenantId, CustomerId customerId, AlarmCountQuery query, Collection orderedEntityIds) { QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, null, EntityType.ALARM)); if (query.isSearchPropagatedAlarms()) { @@ -326,6 +327,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } + if (!orderedEntityIds.isEmpty()) { + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); + } } else { ctx.append("select count(id) from alarm_info a "); ctx.append("where a.tenant_id = :tenantId"); @@ -334,6 +339,10 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } + if (!orderedEntityIds.isEmpty()) { + ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); + } } long startTs; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java index aa1c96d84b..d12765717a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AlarmServiceTest.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; @@ -48,6 +49,7 @@ import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; @@ -936,4 +938,53 @@ public class AlarmServiceTest extends AbstractServiceTest { Assert.assertEquals(0, alarms.getData().size()); } + @Test + public void testCountAlarmsForEntities() throws ExecutionException, InterruptedException { + AssetId parentId = new AssetId(Uuids.timeBased()); + AssetId childId = new AssetId(Uuids.timeBased()); + + EntityRelation relation = new EntityRelation(parentId, childId, EntityRelation.CONTAINS_TYPE); + + Assert.assertTrue(relationService.saveRelationAsync(tenantId, relation).get()); + + long ts = System.currentTimeMillis(); + AlarmApiCallResult result = alarmService.createAlarm(AlarmCreateOrUpdateActiveRequest.builder() + .tenantId(tenantId) + .originator(childId) + .type(TEST_ALARM) + .severity(AlarmSeverity.CRITICAL) + .startTs(ts).build()); + AlarmInfo created = result.getAlarm(); + created.setPropagate(true); + result = alarmService.updateAlarm(AlarmUpdateRequest.fromAlarm(created)); + created = result.getAlarm(); + + EntityListFilter entityListFilter = new EntityListFilter(); + entityListFilter.setEntityList(List.of(childId.getId().toString(), parentId.getId().toString())); + entityListFilter.setEntityType(EntityType.ASSET); + AlarmCountQuery countQuery = new AlarmCountQuery(entityListFilter); + countQuery.setStartTs(0L); + countQuery.setEndTs(System.currentTimeMillis()); + + long alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + + countQuery.setSearchPropagatedAlarms(true); + + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(parentId)); + Assert.assertEquals(1, alarmsCount); + + created = alarmService.acknowledgeAlarm(tenantId, created.getId(), System.currentTimeMillis()).getAlarm(); + + countQuery.setStatusList(List.of(AlarmSearchStatus.UNACK)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(0, alarmsCount); + + alarmService.clearAlarm(tenantId, created.getId(), System.currentTimeMillis(), null); + + countQuery.setStatusList(List.of(AlarmSearchStatus.CLEARED)); + alarmsCount = alarmService.countAlarmsByQuery(tenantId, null, countQuery, List.of(childId)); + Assert.assertEquals(1, alarmsCount); + } + } From 33eac2f77868c4b79c1beb2a99a02fa0047f756c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 10 Feb 2025 17:34:12 +0200 Subject: [PATCH 171/438] fixed tests --- .../service/subscription/TbAlarmCountSubCtx.java | 15 ++++++++------- .../server/common/data/query/AlarmCountQuery.java | 6 ++---- .../common/data/query/EntityCountQuery.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 3 +-- .../sql/query/DefaultAlarmQueryRepository.java | 4 ++-- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 8425b2a57a..4771bad716 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -80,14 +80,15 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); - List entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); - if (entityIds.isEmpty()) { - return 0; - } else { - return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); + List entityIds = null; + if (query.getEntityFilter() != null) { + PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + if (data.getData().isEmpty()) { + return 0; + } + entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); } - + return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); } private EntityDataQuery buildEntityDataQuery() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java index bb969a28ce..1bf19a970f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AlarmCountQuery.java @@ -17,9 +17,8 @@ package org.thingsboard.server.common.data.query; import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Getter; +import lombok.Data; import lombok.NoArgsConstructor; -import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -30,8 +29,7 @@ import java.util.List; @Builder @NoArgsConstructor @AllArgsConstructor -@Getter -@Setter +@Data @ToString public class AlarmCountQuery extends EntityCountQuery { private long startTs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java index be9fa4c664..3dd24147e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/EntityCountQuery.java @@ -27,7 +27,7 @@ import java.util.List; public class EntityCountQuery { @Getter - protected EntityFilter entityFilter; + private EntityFilter entityFilter; @Getter protected List keyFilters; diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 14cd65185d..b8a57fdcff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -351,8 +351,7 @@ public class BaseAlarmService extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - return alarmDao.countAlarmsByQuery(tenantId, customerId, query, Collections.emptyList()); + return countAlarmsByQuery(tenantId, customerId, query, null); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index b4cc40fc6d..216fd0d2d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -327,7 +327,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } - if (!orderedEntityIds.isEmpty()) { + if (orderedEntityIds != null) { ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); } @@ -339,7 +339,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { ctx.append(" and a.customer_id = :customerId"); ctx.addUuidParameter("customerId", customerId.getId()); } - if (!orderedEntityIds.isEmpty()) { + if (orderedEntityIds != null) { ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); } From d75bc3ac28e52ec44ef88ad5f238cb3cfac46cd9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 11 Feb 2025 12:47:37 +0200 Subject: [PATCH 172/438] Minor improvements --- .../cf/ctx/state/RocksDBCalculatedFieldStateService.java | 5 ++--- .../java/org/thingsboard/server/queue/util/AfterStartUp.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 48e17954ea..3374eb2660 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -47,7 +47,7 @@ import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) +@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { private final ActorSystemContext actorSystemContext; @@ -66,12 +66,11 @@ public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateS restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); } - @Override public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { CalculatedFieldStateProto stateProto = toProto(stateId, state); long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= ctx.getMaxStateSizeInKBytes()) { + if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { rocksDBService.put(toProto(stateId), stateProto); } callback.onSuccess(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index f97c3b0a1b..dea6d9ed9e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -41,7 +41,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; int CF_STATE_RESTORE_SERVICE = 12; - int CF_CONSUMER_SERVICE = 13; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From 9e19fab11dda76ebf65c88ac77c311ee175b6bc2 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:43:13 +0200 Subject: [PATCH 173/438] Review comments resolving and refactoring --- .../core/http/calculated-fields.service.ts | 6 +-- .../src/app/core/http/rule-chain.service.ts | 2 +- .../calculated-fields-table-config.ts | 44 ++++++++++--------- ...lculated-field-debug-dialog.component.html | 2 +- ...calculated-field-debug-dialog.component.ts | 10 +++-- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.ts | 29 +++++++----- ...ed-field-script-test-dialog.component.html | 24 +++++----- ...ed-field-script-test-dialog.component.scss | 22 ++++++++++ ...ated-field-script-test-dialog.component.ts | 14 +++--- .../shared/models/calculated-field.models.ts | 15 +++---- ui-ngx/src/app/shared/models/entity.models.ts | 6 +-- .../src/app/shared/models/rule-node.models.ts | 4 +- .../en_US/calculated-field/expression_fn.md | 1 + .../calculated-field/test-expression_fn.md | 1 + 15 files changed, 109 insertions(+), 73 deletions(-) create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md create mode 100644 ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md diff --git a/ui-ngx/src/app/core/http/calculated-fields.service.ts b/ui-ngx/src/app/core/http/calculated-fields.service.ts index acaf3b2817..3e0e08f8e6 100644 --- a/ui-ngx/src/app/core/http/calculated-fields.service.ts +++ b/ui-ngx/src/app/core/http/calculated-fields.service.ts @@ -22,7 +22,7 @@ import { PageData } from '@shared/models/page/page-data'; import { CalculatedField, CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; import { PageLink } from '@shared/models/page/page-link'; import { EntityId } from '@shared/models/id/entity-id'; -import { TestScriptResult } from '@shared/models/entity.models'; +import { EntityTestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' @@ -50,7 +50,7 @@ export class CalculatedFieldsService { defaultHttpOptionsFromConfig(config)); } - public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { - return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); + public testScript(inputParams: CalculatedFieldTestScriptInputParams, config?: RequestConfig): Observable { + return this.http.post('/api/calculatedField/testScript', inputParams, defaultHttpOptionsFromConfig(config)); } } diff --git a/ui-ngx/src/app/core/http/rule-chain.service.ts b/ui-ngx/src/app/core/http/rule-chain.service.ts index c1333df1ac..e3353989cc 100644 --- a/ui-ngx/src/app/core/http/rule-chain.service.ts +++ b/ui-ngx/src/app/core/http/rule-chain.service.ts @@ -35,6 +35,7 @@ import { RuleNodeConfiguration, ScriptLanguage, TestScriptInputParams, + TestScriptResult } from '@app/shared/models/rule-node.models'; import { componentTypeBySelector, ResourcesService } from '../services/resources.service'; import { catchError, map, mergeMap } from 'rxjs/operators'; @@ -43,7 +44,6 @@ import { deepClone, snakeCase } from '@core/utils'; import { DebugRuleNodeEventBody } from '@app/shared/models/event.models'; import { Edge } from '@shared/models/edge.models'; import { IModulesMap } from '@modules/common/modules-map.models'; -import { TestScriptResult } from '@shared/models/entity.models'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c71eef8de3..63b0fad732 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -34,12 +34,12 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { TbPopoverService } from '@shared/components/popover.service'; import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug/entity-debug-settings-panel.component'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; -import { catchError, filter, switchMap } from 'rxjs/operators'; +import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldScriptTestDialogData + CalculatedFieldTestScriptInputParams, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -47,7 +47,6 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; -import { CalculatedFieldId } from '@shared/models/id/calculated-field-id'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +57,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, id, expression), + action: (calculatedField: CalculatedField) => this.openDebugDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -139,10 +138,11 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(id, configuration?.expression) + action: () => this.openDebugDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -178,8 +178,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.calculatedFieldsService.saveCalculatedField({ ...calculatedField, ...updatedCalculatedField })), @@ -191,7 +191,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { + private getCalculatedFieldDialog(value?: CalculatedField, buttonTitle = 'action.add', isDirty = false): Observable { return this.dialog.open(CalculatedFieldDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -204,21 +204,20 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { tenantId: this.tenantId, - entityId: this.entityId, - id, - expression, - testScriptFn: this.getTestScriptDialog.bind(this), + value: calculatedField, + getTestScriptDialogFn: this.getTestScriptDialog.bind(this), } }) .afterClosed() @@ -260,16 +259,21 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(argumentsObj: Record, expression: string, withApply = false): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj, - expression, - withApply, + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => ({...acc, [key]: '' }), {}), + expression: calculatedField.configuration.expression, } - }).afterClosed(); + }).afterClosed() + .pipe( + filter(Boolean), + tap(expression => + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + ), + ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index f88176be8a..8295a9c892 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -32,7 +32,7 @@ [disabledEventTypes]="[EventType.LC_EVENT, EventType.ERROR, EventType.STATS]" [defaultEventType]="DebugEventType.DEBUG_CALCULATED_FIELD" [active]="true" - [entityId]="data.id" + [entityId]="data.value.id" [functionTestButtonLabel]="'common.test-function' | translate" (debugEventSelected)="onDebugEventSelected($event)" /> diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index 5b79476528..be27e277a6 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -22,14 +22,14 @@ import { Router } from '@angular/router'; import { DialogComponent } from '@shared/components/dialog.component'; import { CalculatedFieldEventBody, DebugEventType, EventType } from '@shared/models/event.models'; import { EventTableComponent } from '@home/components/event/event-table.component'; -import { CalculatedFieldDebugDialogData } from '@shared/models/calculated-field.models'; +import { CalculatedFieldDebugDialogData, CalculatedFieldType } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-debug-dialog', styleUrls: ['calculated-field-debug-dialog.component.scss'], templateUrl: './calculated-field-debug-dialog.component.html', }) -export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { +export class CalculatedFieldDebugDialogComponent extends DialogComponent implements AfterViewInit { @ViewChild(EventTableComponent, {static: true}) eventsTable: EventTableComponent; @@ -40,12 +40,13 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent, protected router: Router, @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldDebugDialogData, - protected dialogRef: MatDialogRef) { + protected dialogRef: MatDialogRef) { super(store, router, dialogRef); } ngAfterViewInit(): void { this.eventsTable.entitiesTable.updateData(); + this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = () => this.data.value.type === CalculatedFieldType.SCRIPT; } cancel(): void { @@ -53,6 +54,7 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent this.dialogRef.close(expression)); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index ad2d14cbbf..f50c552ec0 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,7 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" - helpId="[TODO]: [Calculated Fields] add valid link" + helpId="calculated-field/expression_fn" />
- @if (data.withApply) { - - } +
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index ee0d59b839..2420aa5a50 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -27,6 +27,28 @@ padding-left: 5px; border: 1px solid #c0c0c0; } + + .block-label-container { + position: absolute; + z-index: 10; + font-size: 12px; + font-weight: bold; + + &.left { + right: 112px; + top: 9px; + } + + &.right-bottom { + right: 40px; + top: 6px; + } + + &.right-top { + right: 8px; + top: 2px; + } + } } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index fa66827eb6..bd19a68045 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -22,7 +22,7 @@ import { Inject, ViewChild, } from '@angular/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormBuilder } from '@angular/forms'; @@ -36,8 +36,8 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; -import { CalculatedFieldScriptTestDialogData } from '@shared/models/calculated-field.models'; import { filter } from 'rxjs/operators'; +import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,8 +66,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldScriptTestDialogData, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, protected dialogRef: MatDialogRef, + private dialog: MatDialog, private fb: FormBuilder, private destroyRef: DestroyRef, private calculatedFieldService: CalculatedFieldsService) { @@ -101,13 +102,13 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent { + this.testScript(true).pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { this.calculatedFieldScriptTestFormGroup.get('expression').markAsPristine(); this.dialogRef.close(this.calculatedFieldScriptTestFormGroup.get('expression').value); }); } - private testScript(): Observable { + private testScript(onSave = false): Observable { if (this.checkInputParamErrors()) { return this.calculatedFieldService.testScript({ expression: this.calculatedFieldScriptTestFormGroup.get('expression').value, @@ -122,6 +123,9 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, expression: string, withApply?: boolean) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -136,20 +136,15 @@ export interface CalculatedFieldDialogData { debugLimitsConfiguration: string; tenantId: string; entityName?: string; - additionalDebugActionConfig: AdditionalDebugActionConfig; + additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; testScriptFn: CalculatedFieldTestScriptFn; + isDirty?: boolean; } export interface CalculatedFieldDebugDialogData { - id?: CalculatedFieldId; - entityId: EntityId; tenantId: string; - expression?: string; - testScriptFn: CalculatedFieldTestScriptFn; -} - -export interface CalculatedFieldScriptTestDialogData extends CalculatedFieldTestScriptInputParams { - withApply: boolean; + value: CalculatedField; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; } export interface CalculatedFieldTestScriptInputParams { diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 3a8c06f544..472d28f849 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -203,13 +203,13 @@ export interface EntityDebugSettings { allEnabledUntil?: number; } -export interface TestScriptResult { +export interface EntityTestScriptResult { output: string; error: string; } -export interface AdditionalDebugActionConfig { - action?: (id?: EntityId, ...restArguments: unknown[]) => void; +export interface AdditionalDebugActionConfig void> { + action?: Action; title: string; } diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index ac08fa38ac..3e1eb70835 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -27,7 +27,7 @@ import { AppState } from '@core/core.state'; import { AbstractControl, UntypedFormGroup } from '@angular/forms'; import { RuleChainType } from '@shared/models/rule-chain.models'; import { DebugRuleNodeEventBody } from '@shared/models/event.models'; -import { HasEntityDebugSettings } from '@shared/models/entity.models'; +import { EntityTestScriptResult, HasEntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; export interface RuleNodeConfiguration { @@ -374,6 +374,8 @@ export interface TestScriptInputParams { msgType: string; } +export type TestScriptResult = EntityTestScriptResult; + export enum MessageType { POST_ATTRIBUTES_REQUEST = 'POST_ATTRIBUTES_REQUEST', POST_TELEMETRY_REQUEST = 'POST_TELEMETRY_REQUEST', diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/expression_fn.md @@ -0,0 +1 @@ + diff --git a/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md new file mode 100644 index 0000000000..f8173dc528 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/calculated-field/test-expression_fn.md @@ -0,0 +1 @@ + From 2833164968252d1011a3d64d53dbb3d73650c106 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 15:48:45 +0200 Subject: [PATCH 174/438] Refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 2 +- .../components/dialog/calculated-field-dialog.component.ts | 2 +- ui-ngx/src/app/shared/models/calculated-field.models.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 63b0fad732..544860cca2 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -203,7 +203,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { this.configFormGroup.get('expressionSCRIPT').setValue(expression); this.configFormGroup.get('expressionSCRIPT').markAsDirty(); diff --git a/ui-ngx/src/app/shared/models/calculated-field.models.ts b/ui-ngx/src/app/shared/models/calculated-field.models.ts index cec8fe5cd6..19b53bc91e 100644 --- a/ui-ngx/src/app/shared/models/calculated-field.models.ts +++ b/ui-ngx/src/app/shared/models/calculated-field.models.ts @@ -137,7 +137,7 @@ export interface CalculatedFieldDialogData { tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; - testScriptFn: CalculatedFieldTestScriptFn; + getTestScriptDialogFn: CalculatedFieldTestScriptFn; isDirty?: boolean; } From f8d88387fa78bb7f8e93efbe759f8e9a40cad9d3 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:04:23 +0200 Subject: [PATCH 175/438] refactoring --- .../calculated-fields/calculated-fields-table-config.ts | 5 +++-- .../dialog/calculated-field-dialog.component.ts | 8 ++------ .../modules/home/components/event/event-table-config.ts | 4 ++-- ui-ngx/src/app/shared/models/entity.models.ts | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 544860cca2..318ffaca66 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -205,7 +205,8 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig ({...acc, [key]: '' }), {}), + arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, } }).afterClosed() diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 8574ee99c2..04e1b539dd 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -33,7 +33,7 @@ import { import { noLeadTrailSpacesRegex } from '@shared/models/regex.constants'; import { AttributeScope } from '@shared/models/telemetry/telemetry.models'; import { EntityType } from '@shared/models/entity-type.models'; -import { filter, map, startWith } from 'rxjs/operators'; +import { map, startWith } from 'rxjs/operators'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { ScriptLanguage } from '@shared/models/rule-node.models'; @@ -121,11 +121,7 @@ export class CalculatedFieldDialogComponent extends DialogComponent { - this.configFormGroup.get('expressionSCRIPT').setValue(expression); - this.configFormGroup.get('expressionSCRIPT').markAsDirty(); - }); + this.data.getTestScriptDialogFn(this.fromGroupValue).subscribe(); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 39fb143de0..98d2fbefaf 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -360,7 +360,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[1].width = '20%'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', - (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), @@ -380,7 +380,7 @@ export class EventTableConfig extends EntityTableConfig { } ), new EntityTableColumn('messageId', 'event.message-id', '85px', - (entity) => `${entity.body.msgId?.substring(0, 8)}…`, + (entity) => `${entity.body.msgId?.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), diff --git a/ui-ngx/src/app/shared/models/entity.models.ts b/ui-ngx/src/app/shared/models/entity.models.ts index 472d28f849..20a0ccbe77 100644 --- a/ui-ngx/src/app/shared/models/entity.models.ts +++ b/ui-ngx/src/app/shared/models/entity.models.ts @@ -209,7 +209,7 @@ export interface EntityTestScriptResult { } export interface AdditionalDebugActionConfig void> { - action?: Action; + action: Action; title: string; } From 9589317e9750bea3190798c8060deb522ada1e85 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:25:01 +0200 Subject: [PATCH 176/438] preserve edit dialog on test --- .../calculated-fields-table-config.ts | 15 +++++++++------ .../dialog/calculated-field-dialog.component.ts | 5 ++++- ...lculated-field-script-test-dialog.component.ts | 6 +++--- .../app/shared/models/calculated-field.models.ts | 6 +++++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 318ffaca66..966ac5f008 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -39,7 +39,7 @@ import { CalculatedField, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, - CalculatedFieldTestScriptInputParams, + CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { CalculatedFieldDebugDialogComponent, @@ -260,21 +260,24 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record): Observable { - return this.dialog.open(CalculatedFieldScriptTestDialogComponent, + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), expression: calculatedField.configuration.expression, + openCalculatedFieldEdit } }).afterClosed() .pipe( filter(Boolean), - tap(expression => - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) - ), + tap(expression => { + if (openCalculatedFieldEdit) { + this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + } + }), ); } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 04e1b539dd..e75c962268 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -121,7 +121,10 @@ export class CalculatedFieldDialogComponent extends DialogComponent { + this.configFormGroup.get('expressionSCRIPT').setValue(expression); + this.configFormGroup.get('expressionSCRIPT').markAsDirty(); + }); } private applyDialogData(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts index bd19a68045..94ffeeea52 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.ts @@ -37,7 +37,7 @@ import { beautifyJs } from '@shared/models/beautify.models'; import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; import { filter } from 'rxjs/operators'; -import { CalculatedFieldTestScriptInputParams } from '@shared/models/calculated-field.models'; +import { CalculatedFieldTestScriptDialogData } from '@shared/models/calculated-field.models'; @Component({ selector: 'tb-calculated-field-script-test-dialog', @@ -66,7 +66,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent, protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptInputParams, + @Inject(MAT_DIALOG_DATA) public data: CalculatedFieldTestScriptDialogData, protected dialogRef: MatDialogRef, private dialog: MatDialog, private fb: FormBuilder, @@ -123,7 +123,7 @@ export class CalculatedFieldScriptTestDialogComponent extends DialogComponent) => Observable; +export type CalculatedFieldTestScriptFn = (calculatedField: CalculatedField, argumentsObj?: Record, closeAllOnSave?: boolean) => Observable; export interface CalculatedFieldDialogData { value?: CalculatedField; @@ -152,6 +152,10 @@ export interface CalculatedFieldTestScriptInputParams { expression: string; } +export interface CalculatedFieldTestScriptDialogData extends CalculatedFieldTestScriptInputParams { + openCalculatedFieldEdit?: boolean; +} + export interface ArgumentEntityTypeParams { title: string; entityType: EntityType From 6eeaecf88ff69c5ca5de32867dc29e0be6d1456e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 16:30:24 +0200 Subject: [PATCH 177/438] close debug panel on additional action --- .../entity/debug/entity-debug-settings-panel.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html index dcc3355890..f7fd9f737b 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-panel.component.html @@ -54,7 +54,7 @@ } From 934a8daa4ea59b6132afcd48fd781f9812c512dc Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 17:29:44 +0200 Subject: [PATCH 178/438] only last arguments fix --- .../calculated-fields/calculated-fields-table-config.ts | 9 ++++++++- .../modules/home/components/event/event-table-config.ts | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 966ac5f008..9560f264b5 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -261,12 +261,19 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { + const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); + const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { + if (emptyArguments.hasOwnProperty(key)) { + acc[key] = argumentsObj[key]; + } + return acc; + }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { - arguments: argumentsObj ?? Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}), + arguments: { ...emptyArguments, ...filledArguments }, expression: calculatedField.configuration.expression, openCalculatedFieldEdit } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 98d2fbefaf..64917b23c0 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -357,7 +357,7 @@ export class EventTableConfig extends EntityTableConfig { break; case DebugEventType.DEBUG_CALCULATED_FIELD: this.columns[0].width = '80px'; - this.columns[1].width = '20%'; + this.columns[1].width = '100px'; this.columns.push( new EntityTableColumn('entityId', 'event.entity-id', '85px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, From a1b9e941e5c11f9022479ea0e0ce52a58041df0c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Tue, 11 Feb 2025 18:14:38 +0200 Subject: [PATCH 179/438] fixed debugging dialog height --- .../calculated-field-debug-dialog.component.html | 4 ++-- .../calculated-field-debug-dialog.component.scss | 9 +++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 8295a9c892..2397162b05 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
Date: Tue, 11 Feb 2025 18:40:25 +0200 Subject: [PATCH 180/438] amde arguments not required --- .../calculated-field-test-arguments.component.html | 2 +- .../src/app/shared/components/value-input.component.html | 8 ++++---- ui-ngx/src/app/shared/components/value-input.component.ts | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index e6b62a6862..37f6ba4d27 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/shared/components/value-input.component.html b/ui-ngx/src/app/shared/components/value-input.component.html index e6138dd321..a7108f9174 100644 --- a/ui-ngx/src/app/shared/components/value-input.component.html +++ b/ui-ngx/src/app/shared/components/value-input.component.html @@ -32,7 +32,7 @@ - - -
- Date: Tue, 11 Feb 2025 19:06:05 +0200 Subject: [PATCH 181/438] refactoring --- .../calculated-fields-table-config.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index 9560f264b5..ce47292397 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -47,6 +47,7 @@ import { CalculatedFieldScriptTestDialogComponent } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; +import { isObject } from '@core/utils'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -206,7 +207,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig, openCalculatedFieldEdit = true): Observable { - const emptyArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = ''; return acc; }, {}); - const filledArguments = Object.keys(argumentsObj ?? {}).reduce((acc, key) => { - if (emptyArguments.hasOwnProperty(key)) { - acc[key] = argumentsObj[key]; - } + const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { + acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; }, {}); return this.dialog.open(CalculatedFieldScriptTestDialogComponent, @@ -273,7 +271,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig Date: Tue, 11 Feb 2025 19:10:55 +0200 Subject: [PATCH 182/438] WIP: arguments --- .../CalculatedFieldTbelScriptEngine.java | 1 + .../ctx/state/ScriptCalculatedFieldState.java | 25 +++++++++- .../api/tbel/DefaultTbelInvokeService.java | 9 ++-- .../thingsboard/script/api/tbel/TbCfArg.java | 22 +++++++++ .../script/api/tbel/TbCfSingleValueArg.java | 30 ++++++++++++ .../script/api/tbel/TbCfTsRollingArg.java | 47 +++++++++++++++++++ 6 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java create mode 100644 common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java index 9e05e05970..6dcbf33067 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldTbelScriptEngine.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import io.jsonwebtoken.lang.Collections; import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index 4a24d13c93..7d86e376a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -21,12 +21,18 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; +import org.mvel2.execution.ExecutionArrayList; +import org.thingsboard.script.api.tbel.TbCfArg; +import org.thingsboard.script.api.tbel.TbCfSingleValueArg; +import org.thingsboard.script.api.tbel.TbCfTsRollingArg; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.service.cf.CalculatedFieldResult; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -62,8 +68,9 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { } }); Object[] args = ctx.getArgNames().stream() - .map(key -> arguments.get(key).getValue()) + .map(this::toTbelArgument) .toArray(); + ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); Output output = ctx.getOutput(); return Futures.transform(resultFuture, @@ -72,4 +79,20 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { ); } + private TbCfArg toTbelArgument(String key) { + ArgumentEntry argEntry = arguments.get(key); + if (argEntry instanceof SingleValueArgumentEntry svArg) { + return new TbCfSingleValueArg(svArg.getTs(), argEntry.getValue()); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArg) { + var tsRecords = rollingArg.getTsRecords(); + List values = new ArrayList<>(tsRecords.size()); + for(var e : tsRecords.entrySet()){ + values.add(new TbCfSingleValueArg(e.getKey(), e.getValue().getValue())); + } + return new TbCfTsRollingArg(values); + } else { + throw new RuntimeException("Argument is not supported for TBEL execution!"); + } + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 74d6a4e6f5..906e6ac393 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -32,6 +32,7 @@ import org.mvel2.MVEL; import org.mvel2.ParserContext; import org.mvel2.SandboxedParserConfiguration; import org.mvel2.ScriptMemoryOverflowException; +import org.mvel2.integration.PropertyHandlerFactory; import org.mvel2.optimizers.OptimizerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -130,9 +131,11 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); parserConfig = ParserContext.enableSandboxedMode(); parserConfig.addImport("JSON", TbJson.class); - parserConfig.registerDataType("Date", TbDate.class, date -> 8L); - parserConfig.registerDataType("Random", Random.class, date -> 8L); - parserConfig.registerDataType("Calendar", Calendar.class, date -> 8L); + parserConfig.registerDataType("Date", TbDate.class, val -> 8L); + parserConfig.registerDataType("Random", Random.class, val -> 8L); + parserConfig.registerDataType("Calendar", Calendar.class, val -> 8L); + parserConfig.registerDataType("TbCfSingleValueArg", TbCfSingleValueArg.class, TbCfSingleValueArg::memorySize); + parserConfig.registerDataType("TbCfTsRollingArg", TbCfTsRollingArg.class, TbCfTsRollingArg::memorySize); TbUtils.register(parserConfig); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "tbel-executor")); try { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java new file mode 100644 index 0000000000..f5d990a553 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfArg.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +public interface TbCfArg { + + long memorySize(); + +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java new file mode 100644 index 0000000000..e42fb64ba4 --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfSingleValueArg.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Data; + +@Data +public class TbCfSingleValueArg implements TbCfArg { + + private final long ts; + private final Object value; + + @Override + public long memorySize() { + return 8L; // TODO; + } +} diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java new file mode 100644 index 0000000000..a8abbc64ac --- /dev/null +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbCfTsRollingArg.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.script.api.tbel; + +import lombok.Getter; + +import java.util.List; + +public class TbCfTsRollingArg implements TbCfArg { + + @Getter + private final List values; + + public TbCfTsRollingArg(List values) { + this.values = values; + } + + @Override + public long memorySize() { + return values.size() * 8L; //TODO; + } + + public double max() { + double max = Double.MIN_VALUE; + for (TbCfSingleValueArg arg : values) { + double val = Double.valueOf(arg.getValue().toString()); + if (max < val) { + max = val; + } + } + return max; + } + +} From 7b6bb8a6d61546a82305c51862ee087116b10827 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 12 Feb 2025 12:34:56 +0200 Subject: [PATCH 183/438] UI: Add support for entity alias filtering in alarm count data sources --- .../server/service/subscription/TbAlarmCountSubCtx.java | 2 +- .../basic/alarm/alarm-count-basic-config.component.html | 9 +++++++++ .../basic/alarm/alarm-count-basic-config.component.ts | 2 ++ .../components/widget/config/datasource.component.html | 8 ++++---- .../components/widget/config/datasource.component.ts | 9 +++++++++ .../components/widget/config/datasources.component.ts | 4 ++++ 6 files changed, 29 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 4771bad716..2a5dea70ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -94,6 +94,6 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx + +
alarm.filter
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts index 4ad664fae3..d365ed3f2d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/alarm/alarm-count-basic-config.component.ts @@ -69,6 +69,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { const settings: CountWidgetSettings = {...countDefaultSettings(true), ...(configData.config.settings || {})}; this.alarmCountWidgetConfigForm = this.fb.group({ alarmFilterConfig: [getAlarmFilterConfig(configData.config.datasources), []], + datasources: [configData.config.datasources, []], settings: [settings, []], @@ -81,6 +82,7 @@ export class AlarmCountBasicConfigComponent extends BasicWidgetConfigComponent { } protected prepareOutputConfig(config: any): WidgetConfigComponentData { + this.widgetConfig.config.datasources = config.datasources; setAlarmFilterConfig(config.alarmFilterConfig, this.widgetConfig.config.datasources); this.widgetConfig.config.settings = {...(this.widgetConfig.config.settings || {}), ...config.settings}; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html index 8af7753525..6c5c5cf083 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/datasource.component.html @@ -36,7 +36,7 @@ datasourceFormGroup.get('type').value === datasourceType.entity || datasourceFormGroup.get('type').value === datasourceType.entityCount || datasourceFormGroup.get('type').value === datasourceType.alarmCount ? datasourceFormGroup.get('type').value : ''"> - @@ -98,7 +98,7 @@ Date: Wed, 12 Feb 2025 15:38:25 +0200 Subject: [PATCH 184/438] fetch entities only on subs creation --- ...efaultTbEntityDataSubscriptionService.java | 10 +- .../subscription/TbAlarmCountSubCtx.java | 106 ++++++++++++++---- .../server/controller/WebsocketApiTest.java | 4 +- .../query/DefaultAlarmQueryRepository.java | 36 +++--- 4 files changed, 117 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index 9812872bf1..b09f8350f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -424,8 +424,12 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc long start = System.currentTimeMillis(); ctx.fetchData(); long end = System.currentTimeMillis(); - stats.getAlarmQueryInvocationCnt().incrementAndGet(); - stats.getAlarmQueryTimeSpent().addAndGet(end - start); + stats.getRegularQueryInvocationCnt().incrementAndGet(); + stats.getRegularQueryTimeSpent().addAndGet(end - start); + ctx.cancelTasks(); + ctx.clearAlarmSubscriptions(); + ctx.fetchAlarmCount(); + ctx.createAlarmSubscriptions(); TbAlarmCountSubCtx finalCtx = ctx; ScheduledFuture task = scheduler.scheduleWithFixedDelay( () -> refreshDynamicQuery(finalCtx), @@ -549,7 +553,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc private TbAlarmCountSubCtx createSubCtx(WebSocketSessionRef sessionRef, AlarmCountCmd cmd) { Map sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new ConcurrentHashMap<>()); TbAlarmCountSubCtx ctx = new TbAlarmCountSubCtx(serviceId, wsService, entityService, localSubscriptionService, - attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription); + attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription, maxAlarmQueriesPerRefreshInterval); if (cmd.getQuery() != null) { ctx.setAndResolveQuery(cmd.getQuery()); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java index 2a5dea70ac..712fc5c378 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmCountSubCtx.java @@ -36,7 +36,9 @@ import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmCountUpdate; -import java.util.List; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; @Slf4j @ToString(callSuper = true) @@ -44,56 +46,120 @@ public class TbAlarmCountSubCtx extends TbAbstractEntityQuerySubCtx subToEntityIdMap; + + private LinkedHashSet entitiesIds; + private final int maxEntitiesPerAlarmSubscription; + private final int maxAlarmQueriesPerRefreshInterval; + @Getter @Setter private volatile int result; + @Getter + @Setter + private boolean tooManyEntities; + + private int alarmCountInvocationAttempts; + public TbAlarmCountSubCtx(String serviceId, WebSocketService wsService, EntityService entityService, TbLocalSubscriptionService localSubscriptionService, AttributesService attributesService, SubscriptionServiceStatistics stats, AlarmService alarmService, - WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription) { + WebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerAlarmSubscription, int maxAlarmQueriesPerRefreshInterval) { super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId); this.alarmService = alarmService; + this.subToEntityIdMap = new ConcurrentHashMap<>(); this.maxEntitiesPerAlarmSubscription = maxEntitiesPerAlarmSubscription; + this.maxAlarmQueriesPerRefreshInterval = maxAlarmQueriesPerRefreshInterval; + this.entitiesIds = null; + } + + @Override + public void clearSubscriptions() { + clearAlarmSubscriptions(); } @Override public void fetchData() { - result = countAlarms(); - sendWsMsg(new AlarmCountUpdate(cmdId, result)); + resetInvocationCounter(); + if (query.getEntityFilter() != null) { + entitiesIds = new LinkedHashSet<>(); + log.trace("[{}] Fetching data: {}", cmdId, alarmCountInvocationAttempts); + PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); + entitiesIds.clear(); + tooManyEntities = data.hasNext(); + for (EntityData entityData : data.getData()) { + entitiesIds.add(entityData.getEntityId()); + } + } } @Override protected void update() { - int newCount = countAlarms(); - if (newCount != result) { - result = newCount; - sendWsMsg(new AlarmCountUpdate(cmdId, result)); + resetInvocationCounter(); + fetchAlarmCount(); + } + + public void fetchAlarmCount() { + alarmCountInvocationAttempts++; + log.trace("[{}] Fetching alarms: {}", cmdId, alarmCountInvocationAttempts); + if (alarmCountInvocationAttempts <= maxAlarmQueriesPerRefreshInterval) { + doFetchAlarmCount(); + } else { + log.trace("[{}] Ignore alarm count fetch due to rate limit: [{}] of maximum [{}]", cmdId, alarmCountInvocationAttempts, maxAlarmQueriesPerRefreshInterval); } } + private void doFetchAlarmCount() { + result = (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entitiesIds); + sendWsMsg(new AlarmCountUpdate(cmdId, result)); + } + @Override public boolean isDynamic() { return true; } - private int countAlarms() { - List entityIds = null; - if (query.getEntityFilter() != null) { - PageData data = entityService.findEntityDataByQuery(getTenantId(), getCustomerId(), buildEntityDataQuery()); - if (data.getData().isEmpty()) { - return 0; - } - entityIds = data.getData().stream().map(EntityData::getEntityId).toList(); - } - return (int) alarmService.countAlarmsByQuery(getTenantId(), getCustomerId(), query, entityIds); - } - private EntityDataQuery buildEntityDataQuery() { EntityDataPageLink edpl = new EntityDataPageLink(maxEntitiesPerAlarmSubscription, 0, null, new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY))); return new EntityDataQuery(query.getEntityFilter(), edpl, null, null, query.getKeyFilters()); } + + private void resetInvocationCounter() { + alarmCountInvocationAttempts = 0; + } + + public void createAlarmSubscriptions() { + for (EntityId entityId : entitiesIds) { + createAlarmSubscriptionForEntity(entityId); + } + } + + private void createAlarmSubscriptionForEntity(EntityId entityId) { + int subIdx = sessionRef.getSessionSubIdSeq().incrementAndGet(); + subToEntityIdMap.put(subIdx, entityId); + log.trace("[{}][{}][{}] Creating alarms subscription for [{}] ", serviceId, cmdId, subIdx, entityId); + TbAlarmsSubscription subscription = TbAlarmsSubscription.builder() + .serviceId(serviceId) + .sessionId(sessionRef.getSessionId()) + .subscriptionId(subIdx) + .tenantId(sessionRef.getSecurityCtx().getTenantId()) + .entityId(entityId) + .updateProcessor((sub, update) -> fetchAlarmCount()) + .build(); + localSubscriptionService.addSubscription(subscription, sessionRef); + } + + public void clearAlarmSubscriptions() { + if (subToEntityIdMap != null) { + for (Integer subId : subToEntityIdMap.keySet()) { + localSubscriptionService.cancelSubscription(getTenantId(), getSessionId(), subId); + } + subToEntityIdMap.clear(); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index e72e1dc806..d1b8bc0430 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -84,7 +84,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @DaoSqlTest @TestPropertySource(properties = { "server.ws.alarms_per_alarm_status_subscription_cache_size=5", - "server.ws.dynamic_page_link.refresh_interval=2" + "server.ws.dynamic_page_link.refresh_interval=3" }) public class WebsocketApiTest extends AbstractControllerTest { @Autowired @@ -422,7 +422,7 @@ public class WebsocketApiTest extends AbstractControllerTest { alarm = doPost("/api/alarm", alarm, Alarm.class); - update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(3000)); + update = getWsClient().parseAlarmCountReply(getWsClient().waitForUpdate(4000)); Assert.assertEquals(1, update.getCmdId()); Assert.assertEquals(1, update.getCount()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 216fd0d2d4..db8c8ea552 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -321,27 +321,35 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { if (query.isSearchPropagatedAlarms()) { ctx.append("select count(distinct(a.id)) from alarm_info a "); ctx.append(JOIN_ENTITY_ALARMS); - ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); - } if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); - ctx.append(" and ea.entity_id in (:entity_filter_entity_ids)"); + ctx.append("where ea.entity_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId and ea.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId and ea.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } else { ctx.append("select count(id) from alarm_info a "); - ctx.append("where a.tenant_id = :tenantId"); - ctx.addUuidParameter("tenantId", tenantId.getId()); - if (customerId != null && !customerId.isNullUid()) { - ctx.append(" and a.customer_id = :customerId"); - ctx.addUuidParameter("customerId", customerId.getId()); - } if (orderedEntityIds != null) { + if (orderedEntityIds.isEmpty()) { + return 0; + } ctx.addUuidListParameter("entity_filter_entity_ids", orderedEntityIds.stream().map(EntityId::getId).collect(Collectors.toList())); - ctx.append(" and a.originator_id in (:entity_filter_entity_ids)"); + ctx.append("where a.originator_id in (:entity_filter_entity_ids)"); + } else { + ctx.append("where a.tenant_id = :tenantId"); + ctx.addUuidParameter("tenantId", tenantId.getId()); + if (customerId != null && !customerId.isNullUid()) { + ctx.append(" and a.customer_id = :customerId"); + ctx.addUuidParameter("customerId", customerId.getId()); + } } } From e542267295c2818eb87f5f17b40deb16097b4f66 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 15:57:12 +0200 Subject: [PATCH 185/438] Added calculated field responsive styles --- ...lated-field-arguments-table.component.html | 18 ++--- ...lated-field-arguments-table.component.scss | 6 ++ ...lculated-field-debug-dialog.component.html | 4 +- ...lculated-field-debug-dialog.component.scss | 10 ++- .../calculated-field-dialog.component.html | 2 +- ...ulated-field-test-arguments.component.html | 4 +- ...ulated-field-test-arguments.component.scss | 29 ++++++++ ...lculated-field-test-arguments.component.ts | 1 + ...ed-field-script-test-dialog.component.html | 6 +- ...ated-field-script-test-dialog.component.ts | 73 ++++++++++++++----- .../components/event/event-table-config.ts | 8 +- 11 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 5ca80b8214..796fe7c138 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -18,19 +18,19 @@
-
{{ 'calculated-fields.argument-name' | translate }}
-
{{ 'calculated-fields.datasource' | translate }}
-
{{ 'common.type' | translate }}
-
{{ 'entity.key' | translate }}
+
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.datasource' | translate }}
+
{{ 'common.type' | translate }}
+
{{ 'entity.key' | translate }}
@for (group of argumentsFormArray.controls; track group) {
- + -
+
@if (group.get('refEntityId')?.get('id')?.value) { @@ -63,7 +63,7 @@ }
- + @if (group.get('refEntityKey').get('type').value; as type) { @@ -72,9 +72,9 @@ } - + -
+
{{ group.get('refEntityKey').get('key').value }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss index a0c90bba32..e6a48dee58 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.scss @@ -27,4 +27,10 @@ font-size: 14px; } } + + .mat-mdc-standard-chip { + .mdc-evolution-chip__cell--primary, .mdc-evolution-chip__action--primary, .mdc-evolution-chip__text-label { + overflow: hidden; + } + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html index 2397162b05..7da70e180e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.debugging' | translate}}

@@ -25,7 +25,7 @@ close -
+
-
+

{{ 'entity.type-calculated-field' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 37f6ba4d27..03781f1ded 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -19,7 +19,7 @@
{{ 'calculated-fields.arguments' | translate }}
-
{{ 'calculated-fields.argument-name' | translate }}
+
{{ 'calculated-fields.argument-name' | translate }}
{{ 'common.value' | translate }}
@@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss new file mode 100644 index 0000000000..1b2c8670c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host::ng-deep { + .tb-form-table-row { + .argument-value { + .tb-value-type.row { + @media #{constants.$mat-lt-sm} { + width: 100px; + min-width: 100px; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts index d7a33d1a89..c8c9f4e778 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.ts @@ -30,6 +30,7 @@ import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @Component({ selector: 'tb-calculated-field-test-arguments', templateUrl: './calculated-field-test-arguments.component.html', + styleUrls: ['./calculated-field-test-arguments.component.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html index 03582b3066..3d3fe79e4b 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'calculated-fields.test-script-function' | translate }} ({{ 'api-usage.tbel' | translate }})

 -
+
{{ 'common.general' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 4e9819a786..8b3bb899ad 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{ 'calculated-fields.argument-settings' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss new file mode 100644 index 0000000000..cfe7ee9461 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .fixed-title-width { + @media #{constants.$mat-lt-sm} { + min-width: 120px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index a2e5545926..66ce0aeeeb 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -42,6 +42,7 @@ import { MINUTE } from '@shared/models/time/time.models'; @Component({ selector: 'tb-calculated-field-argument-panel', templateUrl: './calculated-field-argument-panel.component.html', + styleUrls: ['./calculated-field-argument-panel.component.scss'] }) export class CalculatedFieldArgumentPanelComponent implements OnInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 03781f1ded..08d023591a 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss index 2420aa5a50..eaee8e443d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-dialog/calculated-field-script-test-dialog.component.scss @@ -26,6 +26,7 @@ padding-top: 5px; padding-left: 5px; border: 1px solid #c0c0c0; + overflow: scroll; } .block-label-container { From 8b3fc83acc140ea33e415d741ba5c7fe5f9aed6c Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:07:48 +0200 Subject: [PATCH 191/438] style fixes --- .../calculated-field-dialog.component.html | 2 +- .../calculated-field-dialog.component.scss | 29 +++++++++++++++++++ .../calculated-field-dialog.component.ts | 1 + ...ulated-field-argument-panel.component.html | 2 +- ...ulated-field-argument-panel.component.scss | 7 +++++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 20245414dd..6ff0977178 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'entity.type-calculated-field' | translate}}

diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss new file mode 100644 index 0000000000..300193a6c1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.scss @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@use '../../../../../../../scss/constants' as constants; + +:host { + .dialog-container { + + @media #{constants.$mat-sm} { + min-width: 526px; + } + + @media #{constants.$mat-gt-sm} { + min-width: 770px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index e75c962268..31f07a1c0d 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -40,6 +40,7 @@ import { ScriptLanguage } from '@shared/models/rule-node.models'; @Component({ selector: 'tb-calculated-field-dialog', templateUrl: './calculated-field-dialog.component.html', + styleUrls: ['./calculated-field-dialog.component.scss'], }) export class CalculatedFieldDialogComponent extends DialogComponent implements AfterViewInit { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 8b3bb899ad..c2fe831204 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+
{{ 'calculated-fields.argument-settings' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index cfe7ee9461..7c4e5c2f5e 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -16,6 +16,13 @@ @use '../../../../../../../scss/constants' as constants; :host { + display: flex; + min-width: 520px; + + @media #{constants.$mat-lt-sm} { + min-width: 320px; + } + .fixed-title-width { @media #{constants.$mat-lt-sm} { min-width: 120px; From 9162acedee26f7ade43a6ad4dce9133de19b481b Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:19:06 +0200 Subject: [PATCH 192/438] style fixes --- .../panel/calculated-field-argument-panel.component.scss | 2 +- .../calculated-field-test-arguments.component.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss index 7c4e5c2f5e..ca6d773262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.scss @@ -20,7 +20,7 @@ min-width: 520px; @media #{constants.$mat-lt-sm} { - min-width: 320px; + min-width: 280px; } .fixed-title-width { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html index 08d023591a..eea3523d00 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/test-arguments/calculated-field-test-arguments.component.html @@ -28,7 +28,7 @@ - +
}
From 1f28c0efe9b8a4f1d0c6f50d0c790dea6c2cfc27 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Wed, 12 Feb 2025 18:40:41 +0200 Subject: [PATCH 193/438] style fixes --- .../calculated-field-arguments-table.component.html | 2 +- .../dialog/calculated-field-dialog.component.scss | 10 ++-------- .../calculated-field-argument-panel.component.scss | 7 ++----- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html index 796fe7c138..77347c6477 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/arguments-table/calculated-field-arguments-table.component.html @@ -80,7 +80,7 @@ -
+
@for (group of argumentsFormArray.controls; track group) { @@ -80,7 +80,7 @@ -
+
-
+
Date: Thu, 13 Feb 2025 12:33:30 +0200 Subject: [PATCH 200/438] fixed entity/message id min width --- .../app/modules/home/components/event/event-table-config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 2f2650d434..b32c5f0050 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -359,7 +359,7 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '85px', + new EntityTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, () => ({padding: '0 12px 0 0'}), false, @@ -379,7 +379,7 @@ export class EventTableConfig extends EntityTableConfig { type: CellActionDescriptorType.COPY_BUTTON } ), - new EntityTableColumn('messageId', 'event.message-id', '85px', + new EntityTableColumn('messageId', 'event.message-id', '100px', (entity) => entity.body.msgId ? `${entity.body.msgId?.substring(0, 8)}…` : '-', () => ({padding: '0 12px 0 0'}), false, From 7232d939d86bbfe368cf955f08cd980efe3030f1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 12:57:26 +0200 Subject: [PATCH 201/438] fixed cf version --- .../server/dao/model/sql/CalculatedFieldEntity.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 64c8e8d5b8..55928dbed1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -31,7 +31,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.BaseVersionedEntity; import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; @@ -52,7 +52,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.DEBUG_SETTINGS; @EqualsAndHashCode(callSuper = true) @Entity @Table(name = CALCULATED_FIELD_TABLE_NAME) -public class CalculatedFieldEntity extends BaseSqlEntity implements BaseEntity { +public class CalculatedFieldEntity extends BaseVersionedEntity implements BaseEntity { @Column(name = CALCULATED_FIELD_TENANT_ID_COLUMN) private UUID tenantId; From 677385e8babc8f263e2b662c6142cd6311d40f4c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 13:01:17 +0200 Subject: [PATCH 202/438] CF states restore from Kafka --- ...CalculatedFieldEntityMessageProcessor.java | 9 +- .../AbstractCalculatedFieldStateService.java | 173 +++++++++++++++++ .../cf/CalculatedFieldStateService.java | 5 + .../cf/CfRocksDb.java} | 30 +-- ...faultCalculatedFieldProcessingService.java | 4 +- .../server/service/cf/RocksDBService.java | 97 ---------- ...aultCalculatedFieldEntityProfileCache.java | 11 +- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 + .../KafkaCalculatedFieldStateService.java | 129 ++++++++++++- .../RocksDBCalculatedFieldStateService.java | 175 +++--------------- ...faultTbCalculatedFieldConsumerService.java | 46 +++-- .../queue/DefaultTbClusterService.java | 9 +- .../DefaultTbRuleEngineConsumerService.java | 5 +- .../consumer/MainQueueConsumerManager.java | 10 +- .../queue/ruleengine/TbQueueConsumerTask.java | 12 +- .../auth/rest/RestAuthenticationProvider.java | 1 + .../thingsboard/server/utils/TbRocksDb.java | 71 +++++++ .../src/main/resources/thingsboard.yml | 8 +- .../server/common/data/DataConstants.java | 1 + .../common/data/util/CollectionsUtil.java | 12 ++ common/proto/src/main/proto/queue.proto | 13 +- .../AbstractTbQueueConsumerTemplate.java | 6 +- .../queue/discovery/HashPartitionService.java | 54 ++++-- .../queue/discovery/PartitionService.java | 2 + .../server/queue/discovery/QueueKey.java | 6 + .../discovery/event/PartitionChangeEvent.java | 6 +- .../server/queue/kafka/KafkaTbQueueMsg.java | 9 +- .../queue/kafka/TbKafkaConsumerTemplate.java | 63 ++++++- .../queue/kafka/TbKafkaProducerTemplate.java | 7 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../InMemoryMonolithQueueFactory.java | 5 +- .../provider/KafkaMonolithQueueFactory.java | 40 ++-- .../KafkaTbRuleEngineQueueFactory.java | 40 ++-- .../provider/TbRuleEngineQueueFactory.java | 6 +- .../server/queue/util/AfterStartUp.java | 9 +- 35 files changed, 679 insertions(+), 405 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java rename application/src/main/java/org/thingsboard/server/{utils/RocksDBConfig.java => service/cf/CfRocksDb.java} (58%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 61f7abb297..c45c4c2ef0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -96,8 +96,13 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldStateRestoreMsg msg) { - log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), msg.getId().cfId()); - states.put(msg.getId().cfId(), msg.getState()); + CalculatedFieldId cfId = msg.getId().cfId(); + log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + if (msg.getState() != null) { + states.put(cfId, msg.getState()); + } else { + states.remove(cfId); + } } public void process(EntityInitCalculatedFieldMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java new file mode 100644 index 0000000000..7df20bc139 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.util.KvProtoUtil; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; +import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; +import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; + +import java.util.Optional; +import java.util.TreeMap; +import java.util.UUID; + +public abstract class AbstractCalculatedFieldStateService implements CalculatedFieldStateService { + + @Autowired + private ActorSystemContext actorSystemContext; + + @Override + public final void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { + CalculatedFieldStateMsgProto stateMsg = toProto(stateId, state); + long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); + if (maxStateSizeInKBytes <= 0 || stateMsg.getSerializedSize() <= maxStateSizeInKBytes) { + doPersist(stateId, stateMsg, callback); + } + } + + protected abstract void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback); + + @Override + public final void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doRemove(stateId, callback); + } + + protected abstract void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback); + + protected void processRestoredState(CalculatedFieldStateMsgProto stateMsg) { + CalculatedFieldEntityCtxId stateId = fromProto(stateMsg.getId()); + CalculatedFieldState state = stateMsg.hasState() ? fromProto(stateMsg.getState()) : null; + actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(stateId, state)); + } + + protected CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return CalculatedFieldEntityCtxIdProto.newBuilder() + .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + protected CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { + TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); + } + + protected CalculatedFieldStateMsgProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { + var stateProto = CalculatedFieldStateProto.newBuilder() + .setType(state.getType().name()); + state.getArguments().forEach((argName, argEntry) -> { + if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { + stateProto.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); + } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { + stateProto.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); + } + }); + return CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .setState(stateProto) + .build(); + } + + protected TransportProtos.SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { + TransportProtos.SingleValueArgumentProto.Builder builder = TransportProtos.SingleValueArgumentProto.newBuilder() + .setArgName(argName); + if (entry != SingleValueArgumentEntry.EMPTY) { + builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); + } + Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); + return builder.build(); + } + + protected TransportProtos.TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { + TransportProtos.TsValueListProto.Builder builder = TransportProtos.TsValueListProto.newBuilder().setKey(argName); + if (entry != TsRollingArgumentEntry.EMPTY) { + entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); + } + return builder.build(); + } + + protected CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { + if (StringUtils.isEmpty(proto.getType())) { + return null; + } + + CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); + + CalculatedFieldState state = switch (type) { + case SIMPLE -> new SimpleCalculatedFieldState(); + case SCRIPT -> new ScriptCalculatedFieldState(); + }; + + proto.getSingleValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); + + if (CalculatedFieldType.SCRIPT.equals(type)) { + proto.getRollingValueArgumentsList().forEach(argProto -> + state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); + } + + return state; + } + + protected SingleValueArgumentEntry fromSingleValueArgumentProto(TransportProtos.SingleValueArgumentProto proto) { + if (!proto.hasValue()) { + return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + } + TransportProtos.TsValueProto tsValueProto = proto.getValue(); + long ts = tsValueProto.getTs(); + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); + return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); + } + + protected TsRollingArgumentEntry fromRollingArgumentProto(TransportProtos.TsValueListProto proto) { + if (proto.getTsValueCount() <= 0) { + return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; + } + TreeMap tsRecords = new TreeMap<>(); + proto.getTsValueList().forEach(tsValueProto -> { + BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); + tsRecords.put(tsValueProto.getTs(), kvEntry); + }); + return new TsRollingArgumentEntry(tsRecords); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index 37211c66c5..05f9dab36b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -16,14 +16,19 @@ package org.thingsboard.server.service.cf; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; +import java.util.Set; + public interface CalculatedFieldStateService { void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback); void removeState(CalculatedFieldEntityCtxId stateId, TbCallback callback); + void restore(Set partitions); + } diff --git a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java rename to application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java index 9c3c02f472..cc63d757a3 100644 --- a/application/src/main/java/org/thingsboard/server/utils/RocksDBConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CfRocksDb.java @@ -13,40 +13,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.cf; import jakarta.annotation.PreDestroy; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; +import org.rocksdb.WriteOptions; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.thingsboard.server.utils.TbRocksDb; @Component -public class RocksDBConfig { +public class CfRocksDb extends TbRocksDb { - @Value("${rocksdb.db_path:${java.io.tmpdir}/rocksdb}") - private String dbPath; - private RocksDB db; - - static { - RocksDB.loadLibrary(); - } - - public RocksDB getDb() throws RocksDBException { - if (db == null) { - Options options = new Options().setCreateIfMissing(true); - db = RocksDB.open(options, dbPath); - } - return db; + public CfRocksDb(@Value("${queue.calculated_fields.rocks_db_path:${user.home}/.rocksdb/cf_states}") String path) throws Exception { + super(path, new Options().setCreateIfMissing(true), new WriteOptions().setSync(true)); } @PreDestroy + @Override public void close() { - if (db != null) { - db.close(); - db = null; - } + super.close(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java index 6b47053f2d..f2c6976cbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldProcessingService.java @@ -70,6 +70,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNot import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; @@ -91,7 +92,6 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @TbRuleEngineComponent @Service @@ -188,7 +188,7 @@ public class DefaultCalculatedFieldProcessingService implements CalculatedFieldP if (broadcast) { broadcasts.add(link); } else { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, link.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, link.entityId()); unicasts.computeIfAbsent(tpi, k -> new ArrayList<>()).add(link); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java deleted file mode 100644 index 7181cc43ed..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ /dev/null @@ -1,97 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf; - -import lombok.extern.slf4j.Slf4j; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; -import org.rocksdb.WriteOptions; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.utils.RocksDBConfig; - -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Map; - -@Service -@Slf4j -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) -public class RocksDBService { - - private final RocksDB db; - private final WriteOptions writeOptions; - - public RocksDBService(RocksDBConfig config) throws RocksDBException { - this.db = config.getDb(); - this.writeOptions = new WriteOptions().setSync(true); - } - - public void put(String key, String value) { - try { - db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value.getBytes(StandardCharsets.UTF_8)); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void put(CalculatedFieldEntityCtxIdProto key, CalculatedFieldStateProto value) { - try { - db.put(writeOptions, key.toByteArray(), value.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to store data to RocksDB", e); - } - } - - public void delete(CalculatedFieldEntityCtxIdProto key) { - try { - db.delete(writeOptions, key.toByteArray()); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - - public String get(String key) { - try { - byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); - return value != null ? new String(value, StandardCharsets.UTF_8) : null; - } catch (RocksDBException e) { - log.error("Failed to retrieve data from RocksDB", e); - return null; - } - } - - public Map getAll() { - Map results = new HashMap<>(); - try (RocksIterator iterator = db.newIterator()) { - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - try { - CalculatedFieldEntityCtxIdProto key = CalculatedFieldEntityCtxIdProto.parseFrom(iterator.key()); - CalculatedFieldStateProto value = CalculatedFieldStateProto.parseFrom(iterator.value()); - results.put(key, value); - } catch (Exception e) { - log.error("Failed to retrieve data from RocksDB", e); - } - } - } - return results; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 9a56d35ac4..b2bb2287fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -21,8 +21,8 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.discovery.HashPartitionService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -30,7 +30,6 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -49,7 +48,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - myPartitions = event.getCalculatedFieldsPartitions().stream() + myPartitions = event.getCfPartitions().stream() .filter(TopicPartitionInfo::isMyPartition) .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); //Naive approach that need to be improved. @@ -58,7 +57,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) .add(profileId, entityId, partition, tpi.isMyPartition()); @@ -66,7 +65,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); var partition = tpi.getPartition().orElse(UNKNOWN); var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); //TODO: make this method atomic; @@ -87,7 +86,7 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { - var tpi = partitionService.resolve(HashPartitionService.CALCULATED_FIELD_QUEUE_KEY, entityId); + var tpi = partitionService.resolve(QueueKey.CF, entityId); return tpi.getPartition().orElse(UNKNOWN); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index 6ce0a11ade..a09a5cd035 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -20,4 +20,9 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; public record CalculatedFieldEntityCtxId(TenantId tenantId, CalculatedFieldId cfId, EntityId entityId) { + + public String toKey() { + return cfId + "_" + entityId; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index a1999d438c..152b1affd9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -15,31 +15,140 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.queue.DefaultTbCalculatedFieldConsumerService.CalculatedFieldQueueConfig; +import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; + +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; @Service @RequiredArgsConstructor -@ConditionalOnExpression("'${zk.enabled:false}'=='true' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-rule-engine')") -public class KafkaCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='kafka'") +public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldStateService { + + private final TbRuleEngineQueueFactory queueFactory; + private final PartitionService partitionService; + + @Value("${queue.calculated_fields.poll_interval:25}") + private long pollInterval; + @Value("${queue.calculated_fields.consumer_per_partition:true}") + private boolean consumerPerPartition; + + private MainQueueConsumerManager, CalculatedFieldQueueConfig> stateConsumer; + private TbKafkaProducerTemplate> stateProducer; + + protected ExecutorService consumersExecutor; + protected ExecutorService mgmtExecutor; + protected ScheduledExecutorService scheduler; - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { + private final AtomicInteger counter = new AtomicInteger(); + + @PostConstruct + private void init() { + this.consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("cf-state-consumer")); + this.mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(Math.max(Runtime.getRuntime().availableProcessors(), 4), "cf-state-mgmt"); + this.scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("cf-state-consumer-scheduler"); + + this.stateConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() + .queueKey(QueueKey.CF_STATES) + .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) + .msgPackProcessor((msgs, consumer, config) -> { + for (TbProtoQueueMsg msg : msgs) { + try { + processRestoredState(msg.getValue()); + } catch (Throwable t) { + log.error("Failed to process state message: {}", msg, t); + } + + int processedMsgCount = counter.incrementAndGet(); + if (processedMsgCount % 10000 == 0) { + log.info("Processed {} calculated field state msgs", processedMsgCount); + } + } + }) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerExecutor(consumersExecutor) + .scheduler(scheduler) + .taskExecutor(mgmtExecutor) + .build(); + this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - callback.onSuccess(); + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF_STATES, stateId.entityId()); + stateProducer.send(tpi, stateId.toKey(), new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto), new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + if (callback != null) { + callback.onSuccess(); + } + } + + @Override + public void onFailure(Throwable t) { + if (callback != null) { + callback.onFailure(t); + } + } + }); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - callback.onSuccess(); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + doPersist(stateId, CalculatedFieldStateMsgProto.newBuilder() + .setId(toProto(stateId)) + .build(), callback); + } + + @Override + public void restore(Set partitions) { + partitions = partitions.stream().map(tpi -> tpi.newByTopic(partitionService.getTopic(QueueKey.CF_STATES))).collect(Collectors.toSet()); + log.info("Restoring calculated field states for partitions: {}", partitions.stream().map(TopicPartitionInfo::getFullTopicName).toList()); + long startTs = System.currentTimeMillis(); + counter.set(0); + + stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update + stateConsumer.awaitStop(0);// consumers should stop on their own because stopWhenRead is true, we just need to wait + + log.info("Restored {} calculated field states in {} ms", counter.get(), System.currentTimeMillis() - startTs); + } + + @PreDestroy + private void preDestroy() { + stateConsumer.stop(); + stateConsumer.awaitStop(); + stateProducer.stop(); + + consumersExecutor.shutdownNow(); + mgmtExecutor.shutdownNow(); + scheduler.shutdownNow(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 3374eb2660..eb7f4818d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -15,174 +15,57 @@ */ package org.thingsboard.server.service.cf.ctx.state; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.RequiredArgsConstructor; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.actors.calculatedField.CalculatedFieldStateRestoreMsg; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import org.thingsboard.server.common.data.id.CalculatedFieldId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.common.util.KvProtoUtil; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldEntityCtxIdProto; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; -import org.thingsboard.server.gen.transport.TransportProtos.SingleValueArgumentProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueListProto; -import org.thingsboard.server.gen.transport.TransportProtos.TsValueProto; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; +import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; +import org.thingsboard.server.service.cf.CfRocksDb; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; -import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.UUID; -import java.util.stream.Collectors; +import java.util.Set; @Service @RequiredArgsConstructor -@ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) // Queue type in mem or Kafka; -public class RocksDBCalculatedFieldStateService implements CalculatedFieldStateService { +@Slf4j +@ConditionalOnExpression("'${queue.type:null}'=='in-memory'") +public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldStateService { - private final ActorSystemContext actorSystemContext; - private final RocksDBService rocksDBService; + private final CfRocksDb cfRocksDb; - public Map restoreStates() { - return rocksDBService.getAll().entrySet().stream() - .collect(Collectors.toMap( - entry -> fromProto(entry.getKey()), - entry -> fromProto(entry.getValue()) - )); - } - - @AfterStartUp(order = AfterStartUp.CF_STATE_RESTORE_SERVICE) - public void initCalculatedFieldStates() { - restoreStates().forEach((k, v) -> actorSystemContext.tell(new CalculatedFieldStateRestoreMsg(k, v))); - } + private Set partitions; @Override - public void persistState(CalculatedFieldCtx ctx, CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { - CalculatedFieldStateProto stateProto = toProto(stateId, state); - long maxStateSizeInKBytes = ctx.getMaxStateSizeInKBytes(); - if (maxStateSizeInKBytes <= 0 || stateProto.getSerializedSize() <= maxStateSizeInKBytes) { - rocksDBService.put(toProto(stateId), stateProto); - } + protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateMsgProto stateMsgProto, TbCallback callback) { + cfRocksDb.put(stateId.toKey(), stateMsgProto.toByteArray()); callback.onSuccess(); } @Override - public void removeState(CalculatedFieldEntityCtxId ctxId, TbCallback callback) { - rocksDBService.delete(toProto(ctxId)); + protected void doRemove(CalculatedFieldEntityCtxId stateId, TbCallback callback) { + cfRocksDb.delete(stateId.toKey()); callback.onSuccess(); } - private CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { - return CalculatedFieldEntityCtxIdProto.newBuilder() - .setTenantIdMSB(ctxId.tenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(ctxId.tenantId().getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) - .setEntityType(ctxId.entityId().getEntityType().name()) - .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) - .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) - .build(); - } - - private CalculatedFieldEntityCtxId fromProto(CalculatedFieldEntityCtxIdProto ctxIdProto) { - TenantId tenantId = TenantId.fromUUID(new UUID(ctxIdProto.getTenantIdMSB(), ctxIdProto.getTenantIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - return new CalculatedFieldEntityCtxId(tenantId, calculatedFieldId, entityId); - } - - private CalculatedFieldStateProto toProto(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state) { - CalculatedFieldStateProto.Builder builder = CalculatedFieldStateProto.newBuilder() - .setId(toProto(stateId)) - .setType(state.getType().name()); - - state.getArguments().forEach((argName, argEntry) -> { - if (argEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - builder.addSingleValueArguments(toSingleValueArgumentProto(argName, singleValueArgumentEntry)); - } else if (argEntry instanceof TsRollingArgumentEntry rollingArgumentEntry) { - builder.addRollingValueArguments(toRollingArgumentProto(argName, rollingArgumentEntry)); - } - }); - - return builder.build(); - } - - private SingleValueArgumentProto toSingleValueArgumentProto(String argName, SingleValueArgumentEntry entry) { - SingleValueArgumentProto.Builder builder = SingleValueArgumentProto.newBuilder() - .setArgName(argName); - - if (entry != SingleValueArgumentEntry.EMPTY) { - builder.setValue(KvProtoUtil.toTsValueProto(entry.getTs(), entry.getKvEntryValue())); - } - - Optional.ofNullable(entry.getVersion()).ifPresent(builder::setVersion); - - return builder.build(); - } - - private TsValueListProto toRollingArgumentProto(String argName, TsRollingArgumentEntry entry) { - TsValueListProto.Builder builder = TsValueListProto.newBuilder().setKey(argName); - - if (entry != TsRollingArgumentEntry.EMPTY) { - entry.getTsRecords().forEach((ts, value) -> builder.addTsValue(KvProtoUtil.toTsValueProto(ts, value))); - } - - return builder.build(); - } - - private CalculatedFieldState fromProto(CalculatedFieldStateProto proto) { - if (StringUtils.isEmpty(proto.getType())) { - return null; - } - - CalculatedFieldType type = CalculatedFieldType.valueOf(proto.getType()); - - CalculatedFieldState state = switch (type) { - case SIMPLE -> new SimpleCalculatedFieldState(); - case SCRIPT -> new ScriptCalculatedFieldState(); - }; - - proto.getSingleValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getArgName(), fromSingleValueArgumentProto(argProto))); - - if (CalculatedFieldType.SCRIPT.equals(type)) { - proto.getRollingValueArgumentsList().forEach(argProto -> - state.getArguments().put(argProto.getKey(), fromRollingArgumentProto(argProto))); - } - - return state; - } - - private SingleValueArgumentEntry fromSingleValueArgumentProto(SingleValueArgumentProto proto) { - if (!proto.hasValue()) { - return (SingleValueArgumentEntry) SingleValueArgumentEntry.EMPTY; + @Override + public void restore(Set partitions) { + if (this.partitions == null) { + this.partitions = partitions; + } else { + return; } - TsValueProto tsValueProto = proto.getValue(); - long ts = tsValueProto.getTs(); - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getArgName(), tsValueProto); - return new SingleValueArgumentEntry(ts, kvEntry, proto.getVersion()); - } - private TsRollingArgumentEntry fromRollingArgumentProto(TsValueListProto proto) { - if (proto.getTsValueCount() <= 0) { - return (TsRollingArgumentEntry) TsRollingArgumentEntry.EMPTY; - } - TreeMap tsRecords = new TreeMap<>(); - proto.getTsValueList().forEach(tsValueProto -> { - BasicKvEntry kvEntry = (BasicKvEntry) KvProtoUtil.fromTsValueProto(proto.getKey(), tsValueProto); - tsRecords.put(tsValueProto.getTs(), kvEntry); + cfRocksDb.forEach((key, value) -> { + try { + processRestoredState(CalculatedFieldStateMsgProto.parseFrom(value)); + } catch (InvalidProtocolBufferException e) { + log.error("[{}] Failed to process restored state", key, e); + } }); - return new TsRollingArgumentEntry(tsRecords); } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index f3e976084b..82ee4de7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -21,11 +21,11 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.calculatedField.CalculatedFieldLinkedTelemetryMsg; import org.thingsboard.server.actors.calculatedField.CalculatedFieldTelemetryMsg; @@ -52,6 +52,7 @@ import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; +import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.consumer.MainQueueConsumerManager; @@ -65,6 +66,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -86,10 +89,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer private int poolSize; private final TbRuleEngineQueueFactory queueFactory; + private final CalculatedFieldStateService stateService; private MainQueueConsumerManager, CalculatedFieldQueueConfig> mainConsumer; - private volatile ListeningExecutorService calculatedFieldsExecutor; + private ListeningExecutorService calculatedFieldsExecutor; + private ExecutorService repartitionExecutor; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -100,19 +105,22 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer PartitionService partitionService, ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, - CalculatedFieldCache calculatedFieldCache) { + CalculatedFieldCache calculatedFieldCache, + CalculatedFieldStateService stateService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; + this.stateService = stateService; } @PostConstruct public void init() { super.init("tb-cf"); this.calculatedFieldsExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(poolSize, "tb-cf-executor")); // TODO: multiple threads. + this.repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-cf-repartition")); this.mainConsumer = MainQueueConsumerManager., CalculatedFieldQueueConfig>builder() - .queueKey(new QueueKey(ServiceType.TB_RULE_ENGINE)) + .queueKey(QueueKey.CF) .config(CalculatedFieldQueueConfig.of(consumerPerPartition, (int) pollInterval)) .msgPackProcessor(this::processMsgs) .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) @@ -137,20 +145,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - var partitions = event.getCalculatedFieldsPartitions(); - log.info("Subscribing to partitions: {}", partitions); - // TODO: @vklimov - before update of the main consumer, we should read the state topics and use - // CalculatedFieldStateService (KafkaCalculatedFieldStateService) to restore the states for entities that belong to new partitions. - // Cleanup entities that do not belong to current partition; - mainConsumer.update(event.getCalculatedFieldsPartitions()); - // Cleanup old entities after corresponding consumers are stopped. - // Any periodic tasks need to check that the entity is still managed by the current server before processing. - actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + var partitions = event.getCfPartitions(); + repartitionExecutor.submit(() -> { + try { + stateService.restore(partitions); + mainConsumer.update(partitions); + // Cleanup old entities after corresponding consumers are stopped. + // Any periodic tasks need to check that the entity is still managed by the current server before processing. + actorContext.tell(new CalculatedFieldPartitionChangeMsg(partitionsToBooleanIndexArray(partitions))); + } catch (Throwable t) { + log.error("Failed to process partition change event: {}", event, t); + } + }); } private boolean[] partitionsToBooleanIndexArray(Set partitions) { boolean[] myPartitions = new boolean[partitionService.getTotalCalculatedFieldPartitions()]; - for(var tpi : partitions) { + for (var tpi : partitions) { tpi.getPartition().ifPresent(partition -> myPartitions[partition] = true); } return myPartitions; @@ -193,9 +204,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerSer packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } - if (log.isDebugEnabled()) { - ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); - } +// if (log.isDebugEnabled()) { +// ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); +// } + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); // TODO: replace with commented above after testing ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 48d26895b9..920a7563dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -94,6 +94,7 @@ import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.TbRuleEngineProducerService; import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; @@ -111,7 +112,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.thingsboard.server.common.util.ProtoUtils.toProto; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; @Service @Slf4j @@ -358,7 +358,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushMsgToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); pushMsgToCalculatedFields(tpi, UUID.randomUUID(), msg, callback); } @@ -371,7 +371,7 @@ public class DefaultTbClusterService implements TbClusterService { @Override public void pushNotificationToCalculatedFields(TenantId tenantId, EntityId entityId, ToCalculatedFieldNotificationMsg msg, TbQueueCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(CALCULATED_FIELD_QUEUE_KEY, entityId); + TopicPartitionInfo tpi = partitionService.resolve(QueueKey.CF, entityId); producerProvider.getCalculatedFieldsNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), callback); toRuleEngineNfs.incrementAndGet(); } @@ -792,8 +792,7 @@ public class DefaultTbClusterService implements TbClusterService { private void pushDeviceUpdateMessage(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { log.trace("{} Going to send edge update notification for device actor, device id {}, edge id {}", tenantId, entityId, edgeId); switch (action) { - case ASSIGNED_TO_EDGE -> - pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); + case ASSIGNED_TO_EDGE -> pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), edgeId), null); case UNASSIGNED_FROM_EDGE -> { EdgeId relatedEdgeId = findRelatedEdgeIdIfAny(tenantId, entityId); pushMsgToCore(new DeviceEdgeUpdateMsg(tenantId, new DeviceId(entityId.getId()), relatedEdgeId), null); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 5e07e33a9e..c42f7c490b 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -63,8 +64,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @Service @TbRuleEngineComponent @Slf4j @@ -109,7 +108,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { event.getPartitionsMap().forEach((queueKey, partitions) -> { - if (CALCULATED_FIELD_QUEUE_KEY.equals(queueKey)) { + if (CollectionsUtil.isOneOf(queueKey, QueueKey.CF, QueueKey.CF_STATES)) { return; } if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java index 6eb5c94c9b..5fa69695d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/consumer/MainQueueConsumerManager.java @@ -182,7 +182,7 @@ public class MainQueueConsumerManager partitions) { + public void doUpdate(Set partitions) { this.partitions = partitions; consumerWrapper.updatePartitions(partitions); } @@ -226,7 +226,9 @@ public class MainQueueConsumerManager msgs, TbQueueConsumer consumer, C config) throws Exception { + log.trace("Processing {} messages", msgs.size()); msgPackProcessor.process(msgs, consumer, config); + log.trace("Processed {} messages", msgs.size()); } public void stop() { @@ -236,8 +238,12 @@ public class MainQueueConsumerManager consumerTask.awaitCompletion(timeoutSec)); log.debug("[{}] Unsubscribed and stopped consumers", queueKey); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java index 5e672eb5c6..0b4e7c02d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbQueueConsumerTask.java @@ -70,13 +70,21 @@ public class TbQueueConsumerTask { } public void awaitCompletion() { + awaitCompletion(30); + } + + public void awaitCompletion(long timeoutSec) { log.trace("[{}] Awaiting finish", key); if (isRunning()) { try { - task.get(30, TimeUnit.SECONDS); + if (timeoutSec > 0) { + task.get(timeoutSec, TimeUnit.SECONDS); + } else { + task.get(); + } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index b9fe54deec..56215b82ac 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,6 +106,7 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { + systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java new file mode 100644 index 0000000000..fa56e0ec1d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/TbRocksDb.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import lombok.SneakyThrows; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksIterator; +import org.rocksdb.WriteOptions; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.BiConsumer; + +public class TbRocksDb { + + protected final String path; + private final WriteOptions writeOptions; + protected final RocksDB db; + + static { + RocksDB.loadLibrary(); + } + + public TbRocksDb(String path, Options dbOptions, WriteOptions writeOptions) throws Exception { + this.path = path; + this.writeOptions = writeOptions; + Files.createDirectories(Path.of(path).getParent()); + this.db = RocksDB.open(dbOptions, path); + } + + @SneakyThrows + public void put(String key, byte[] value) { + db.put(writeOptions, key.getBytes(StandardCharsets.UTF_8), value); + } + + public void forEach(BiConsumer processor) { + try (RocksIterator iterator = db.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + String key = new String(iterator.key(), StandardCharsets.UTF_8); + processor.accept(key, iterator.value()); + } + } + } + + @SneakyThrows + public void delete(String key) { + db.delete(writeOptions, key.getBytes(StandardCharsets.UTF_8)); + } + + public void close() { + if (db != null) { + db.close(); + } + } + +} \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 197f05a5d5..e2570e9137 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -426,10 +426,6 @@ sql: pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls -rocksdb: - # Rocksdb path - db_path: "${ROCKS_DB_PATH:${java.io.tmpdir}/rocksdb}" - # Actor system parameters actors: system: @@ -1629,6 +1625,8 @@ queue: edge-event: "${TB_QUEUE_KAFKA_EDGE_EVENT_TOPIC_PROPERTIES:retention.ms:2592000000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Calculated Field topics calculated-field: "${TB_QUEUE_KAFKA_CF_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Calculated Field State topics + calculated-field-state: "${TB_QUEUE_KAFKA_CF_STATE_TOPIC_PROPERTIES:retention.ms:-1;segment.bytes:52428800;retention.bytes:104857600000;partitions:1;min.insync.replicas:1;cleanup.policy:compact}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1763,6 +1761,8 @@ queue: consumer_per_partition: "${TB_QUEUE_CF_CONSUMER_PER_PARTITION:true}" # Thread pool size for processing of the incoming messages pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" + # RocksDB path for storing CF states + rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 77a7c4a781..c84cae40dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -146,5 +146,6 @@ public class DataConstants { public static final String EDGE_EVENT_QUEUE_NAME = "EdgeEvent"; public static final String CF_QUEUE_NAME = "CalculatedFields"; + public static final String CF_STATES_QUEUE_NAME = "CalculatedFieldStates"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java index e89f3c1ad8..7f9a611c98 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/CollectionsUtil.java @@ -75,4 +75,16 @@ public class CollectionsUtil { return isEmpty(collection) || collection.contains(element); } + public static boolean isOneOf(V value, V... others) { + if (value == null) { + return false; + } + for (V other : others) { + if (value.equals(other)) { + return true; + } + } + return false; + } + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4a2f3710ae..992c6ac42f 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -814,12 +814,15 @@ message SingleValueArgumentProto { int64 version = 3; } -message CalculatedFieldStateProto { +message CalculatedFieldStateMsgProto { CalculatedFieldEntityCtxIdProto id = 1; - // int32 version = 2; - string type = 3; - repeated SingleValueArgumentProto singleValueArguments = 4; - repeated TsValueListProto rollingValueArguments = 5; + CalculatedFieldStateProto state = 2; +} + +message CalculatedFieldStateProto { + string type = 1; + repeated SingleValueArgumentProto singleValueArguments = 2; + repeated TsValueListProto rollingValueArguments = 3; } //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 9513565ca1..6a5b86cf54 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -120,9 +120,9 @@ public abstract class AbstractTbQueueConsumerTemplate i if (record != null) { result.add(decode(record)); } - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - throw new RuntimeException("Failed to decode record: ", e); + } catch (Exception e) { + log.error("Failed to decode record {}", record, e); + throw new RuntimeException("Failed to decode record " + record, e); } }); return result; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index f2f1ccd19c..249dfa859f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -51,7 +51,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.EDGE_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; @Service @Slf4j @@ -78,8 +79,6 @@ public class HashPartitionService implements PartitionService { @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; - public static final QueueKey CALCULATED_FIELD_QUEUE_KEY = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); - private final ApplicationEventPublisher applicationEventPublisher; private final TbServiceInfoProvider serviceInfoProvider; private final TenantRoutingInfoService tenantRoutingInfoService; @@ -120,8 +119,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); - partitionSizesMap.put(CALCULATED_FIELD_QUEUE_KEY, cfPartitions); - partitionTopicsMap.put(CALCULATED_FIELD_QUEUE_KEY, cfEventTopic); + partitionSizesMap.put(QueueKey.CF, cfPartitions); + partitionTopicsMap.put(QueueKey.CF, cfEventTopic); + partitionSizesMap.put(QueueKey.CF_STATES, cfPartitions); + partitionTopicsMap.put(QueueKey.CF_STATES, cfStateTopic); QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); partitionSizesMap.put(vcKey, vcPartitions); @@ -148,6 +149,11 @@ public class HashPartitionService implements PartitionService { return myPartitions.get(queueKey); } + @Override + public String getTopic(QueueKey queueKey) { + return partitionTopicsMap.get(queueKey); + } + private void doInitRuleEnginePartitions() { List queueRoutingInfoList = getQueueRoutingInfos(); queueRoutingInfoList.forEach(queue -> { @@ -222,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); } } @@ -402,6 +408,7 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); + Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -422,19 +429,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - Set tpiList = partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - changedPartitionsMap.put(queueKey, tpiList); + changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); + oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); } }); if (!changedPartitionsMap.isEmpty()) { - Map>> partitionsByServiceType = new HashMap<>(); - changedPartitionsMap.forEach((queueKey, partitions) -> { - partitionsByServiceType.computeIfAbsent(queueKey.getType(), serviceType -> new HashMap<>()) - .put(queueKey, partitions); - }); - partitionsByServiceType.forEach(this::publishPartitionChangeEvent); + changedPartitionsMap.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + .forEach((serviceType, partitionsMap) -> { + publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); + }); } if (currentOtherServices == null) { @@ -466,13 +470,15 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { - log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, + Map> newPartitions, + Map> oldPartitions) { + log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -480,6 +486,15 @@ public class HashPartitionService implements PartitionService { } } + private Set toTpiList(QueueKey queueKey, List partitions) { + if (partitions == null) { + return Collections.emptySet(); + } + return partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + } + @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); @@ -508,7 +523,6 @@ public class HashPartitionService implements PartitionService { return result; } - @Override public int resolvePartitionIndex(UUID entityId, int partitions) { int hash = hash(entityId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index b0e1229f97..5bbff7663c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -45,6 +45,8 @@ public interface PartitionService { List getMyPartitions(QueueKey queueKey); + String getTopic(QueueKey queueKey); + /** * Received from the Discovery service when network topology is changed. * @param currentService - current service information {@link org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index b991e4614c..3f8926ad18 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import static org.thingsboard.server.common.data.DataConstants.CF_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.CF_STATES_QUEUE_NAME; + @Data @AllArgsConstructor public class QueueKey { @@ -32,6 +35,9 @@ public class QueueKey { private final String queueName; private final TenantId tenantId; + public static final QueueKey CF = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_QUEUE_NAME); + public static final QueueKey CF_STATES = new QueueKey(ServiceType.TB_RULE_ENGINE).withQueueName(CF_STATES_QUEUE_NAME); + public QueueKey(ServiceType type, Queue queue) { this.type = type; this.queueName = queue.getName(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 57a4941981..a16d33c884 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -28,8 +28,6 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.thingsboard.server.queue.discovery.HashPartitionService.CALCULATED_FIELD_QUEUE_KEY; - @ToString(callSuper = true) public class PartitionChangeEvent extends TbApplicationEvent { @@ -55,8 +53,8 @@ public class PartitionChangeEvent extends TbApplicationEvent { return getPartitionsByServiceTypeAndQueueName(ServiceType.TB_CORE, DataConstants.EDGE_QUEUE_NAME); } - public Set getCalculatedFieldsPartitions() { - return partitionsMap.getOrDefault(CALCULATED_FIELD_QUEUE_KEY, Collections.emptySet()); + public Set getCfPartitions() { + return partitionsMap.getOrDefault(QueueKey.CF, Collections.emptySet()); } private Set getPartitionsByServiceTypeAndQueueName(ServiceType serviceType, String queueName) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index a2edc35d94..fe2c8b8a90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -23,12 +23,19 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsgHeaders; import java.util.UUID; public class KafkaTbQueueMsg implements TbQueueMsg { + + private static final int UUID_LENGTH = 36; + private final UUID key; private final TbQueueMsgHeaders headers; private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - this.key = UUID.fromString(record.key()); + if (record.key().length() == UUID_LENGTH) { + this.key = UUID.fromString(record.key()); + } else { + this.key = UUID.randomUUID(); + } TbQueueMsgHeaders headers = new DefaultTbQueueMsgHeaders(); record.headers().forEach(header -> { headers.put(header.key(), header.value()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index ef79834735..1d39a3bd0c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; import org.springframework.util.StopWatch; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueMsg; @@ -29,9 +31,12 @@ import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. @@ -46,10 +51,15 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue private final TbKafkaConsumerStatsService statsService; private final String groupId; + private final boolean readFromBeginning; // reset offset to beginning + private final boolean stopWhenRead; // stop consuming when reached initial end offsets + private Map endOffsets; // needed if stopWhenRead is true + @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, String clientId, String groupId, String topic, - TbQueueAdmin admin, TbKafkaConsumerStatsService statsService) { + TbQueueAdmin admin, TbKafkaConsumerStatsService statsService, + boolean readFromBeginning, boolean stopWhenRead) { super(topic); Properties props = settings.toConsumerProps(topic); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); @@ -67,13 +77,35 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; + this.readFromBeginning = readFromBeginning; + this.stopWhenRead = stopWhenRead; } @Override protected void doSubscribe(List topicNames) { if (!topicNames.isEmpty()) { topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); + if (readFromBeginning || stopWhenRead) { + consumer.subscribe(topicNames, new ConsumerRebalanceListener() { + @Override + public void onPartitionsRevoked(Collection partitions) {} + + @Override + public void onPartitionsAssigned(Collection partitions) { + log.debug("Handling onPartitionsAssigned {}", partitions); + if (readFromBeginning) { + consumer.seekToBeginning(partitions); + } + if (stopWhenRead) { + endOffsets = consumer.endOffsets(partitions).entrySet().stream() + .filter(entry -> entry.getValue() > 0) + .collect(Collectors.toMap(entry -> entry.getKey().partition(), Map.Entry::getValue)); + } + } + }); + } else { + consumer.subscribe(topicNames); + } } else { log.info("unsubscribe due to empty topic list"); consumer.unsubscribe(); @@ -92,13 +124,32 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue stopWatch.stop(); log.trace("poll topic {} took {}ms", getTopic(), stopWatch.getTotalTimeMillis()); + List> recordList; if (records.isEmpty()) { - return Collections.emptyList(); + recordList = Collections.emptyList(); } else { - List> recordList = new ArrayList<>(256); - records.forEach(recordList::add); - return recordList; + recordList = new ArrayList<>(256); + records.forEach(record -> { + recordList.add(record); + if (stopWhenRead && endOffsets != null) { + int partition = record.partition(); + Long endOffset = endOffsets.get(partition); + if (endOffset == null) { + log.warn("End offset not found for {} [{}]", record.topic(), partition); + return; + } + log.trace("[{}-{}] Got record offset {}, expected end offset: {}", record.topic(), partition, record.offset(), endOffset - 1); + if (record.offset() >= endOffset - 1) { + endOffsets.remove(partition); + } + } + }); + } + if (stopWhenRead && endOffsets != null && endOffsets.isEmpty()) { + log.info("Reached end offset for {}, stopping consumer", consumer.assignment()); + stop(); } + return recordList; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java index 3c9b85e925..de4fc7049d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaProducerTemplate.java @@ -97,9 +97,12 @@ public class TbKafkaProducerTemplate implements TbQueuePro @Override public void send(TopicPartitionInfo tpi, T msg, TbQueueCallback callback) { + send(tpi, msg.getKey().toString(), msg, callback); + } + + public void send(TopicPartitionInfo tpi, String key, T msg, TbQueueCallback callback) { try { createTopicIfNotExist(tpi); - String key = msg.getKey().toString(); byte[] data = msg.getData(); ProducerRecord record; List
headers = msg.getHeaders().getData().entrySet().stream().map(e -> new RecordHeader(e.getKey(), e.getValue())).collect(Collectors.toList()); @@ -116,7 +119,7 @@ public class TbKafkaProducerTemplate implements TbQueuePro if (callback != null) { callback.onFailure(exception); } else { - log.warn("Producer template failure: {}", exception.getMessage(), exception); + log.warn("Producer template failure", exception); } } }); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index cdd0add38b..7213cf34b8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -54,6 +54,8 @@ public class TbKafkaTopicConfigs { private String housekeeperReprocessingProperties; @Value("${queue.kafka.topic-properties.calculated-field:}") private String calculatedFieldProperties; + @Value("${queue.kafka.topic-properties.calculated-field-state:}") + private String calculatedFieldStateProperties; @Getter private Map coreConfigs; @@ -83,6 +85,8 @@ public class TbKafkaTopicConfigs { private Map edgeEventConfigs; @Getter private Map calculatedFieldConfigs; + @Getter + private Map calculatedFieldStateConfigs; @PostConstruct private void init() { @@ -102,6 +106,7 @@ public class TbKafkaTopicConfigs { edgeConfigs = PropertyUtils.getProps(edgeProperties); edgeEventConfigs = PropertyUtils.getProps(edgeEventProperties); calculatedFieldConfigs = PropertyUtils.getProps(calculatedFieldProperties); + calculatedFieldStateConfigs = PropertyUtils.getProps(calculatedFieldStateProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index c26e2d15c9..8e36ae4962 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; @@ -159,12 +160,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { + public TbQueueConsumer> createCalculatedFieldStateConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { + public TbQueueProducer> createCalculatedFieldStateProducer() { return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 01e174023c..99a50b2f5a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -100,6 +100,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -142,6 +143,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -546,26 +548,28 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 46b35f9acd..4fbf5c3e45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -87,6 +87,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; + private final TbQueueAdmin cfStateAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -119,6 +120,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.edgeAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeConfigs()); this.edgeEventAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); this.cfAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldConfigs()); + this.cfStateAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCalculatedFieldStateConfigs()); } @Override @@ -338,26 +340,28 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateProto.parseFrom(msg.getData()), msg.getHeaders())); - consumerBuilder.admin(cfAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), CalculatedFieldStateMsgProto.parseFrom(msg.getData()), msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override - public TbQueueProducer> createCalculatedFieldStateProducer() { - TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); - requestBuilder.settings(kafkaSettings); - requestBuilder.clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()); - requestBuilder.defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - requestBuilder.admin(cfAdmin); - return requestBuilder.build(); + public TbQueueProducer> createCalculatedFieldStateProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-to-calculated-field-state-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())) + .admin(cfStateAdmin) + .build(); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index d3c1d09399..5f0af72955 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -17,7 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; +import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -126,8 +126,8 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueConsumer> createToCalculatedFieldNotificationsMsgConsumer(); - TbQueueConsumer> createCalculatedFieldStateConsumer(); + TbQueueConsumer> createCalculatedFieldStateConsumer(); - TbQueueProducer> createCalculatedFieldStateProducer(); + TbQueueProducer> createCalculatedFieldStateProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index dea6d9ed9e..a24f0663b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -40,7 +40,6 @@ public @interface AfterStartUp { int CF_READ_PROFILE_ENTITIES_SERVICE = 10; int CF_READ_CF_SERVICE = 11; - int CF_STATE_RESTORE_SERVICE = 12; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; From eae72065ee2dab301efc274e40154ea67302d83d Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 13 Feb 2025 13:59:28 +0200 Subject: [PATCH 203/438] fixed integration tests and added debug events creation --- ...CalculatedFieldEntityMessageProcessor.java | 55 ++-- .../CalculatedFieldStateException.java | 2 +- .../cf/ctx/state/TsRollingArgumentEntry.java | 22 +- .../cf/CalculatedFieldIntegrationTest.java | 284 ++++++++++-------- 4 files changed, 198 insertions(+), 165 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index bfed515eb5..66b32dfbfe 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -197,12 +197,18 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state = getOrInitState(ctx); justRestored = true; } - if (state.updateState(newArgValues) || justRestored) { - cfIdList = new ArrayList<>(cfIdList); - cfIdList.add(ctx.getCfId()); - processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); - } else { - callback.onSuccess(CALLBACKS_PER_CF); + try { + if (state.updateState(newArgValues) || justRestored) { + cfIdList = new ArrayList<>(cfIdList); + cfIdList.add(ctx.getCfId()); + processStateIfReady(ctx, cfIdList, state, tbMsgId, tbMsgType, callback); + } else { + callback.onSuccess(CALLBACKS_PER_CF); + } + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); + } } } @@ -212,37 +218,38 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM if (state != null) { return state; } else { - ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); - // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. - // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. - // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, - // but this will significantly complicate the code. - state = stateFuture.get(1, TimeUnit.MINUTES); - states.put(ctx.getCfId(), state); + try { + ListenableFuture stateFuture = systemContext.getCalculatedFieldProcessingService().fetchStateFromDb(ctx, entityId); + // Ugly but necessary. We do not expect to often fetch data from DB. Only once per pair lifetime. + // This call happens while processing the CF pack from the queue consumer. So the timeout should be relatively low. + // Alternatively, we can fetch the state outside the actor system and push separate command to create this actor, + // but this will significantly complicate the code. + state = stateFuture.get(1, TimeUnit.MINUTES); + states.put(ctx.getCfId(), state); + } catch (Exception e) { + if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { + systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, null, null, null, null, e); + } + throw new RuntimeException(e); + } } return state; } @SneakyThrows private void processStateIfReady(CalculatedFieldCtx ctx, List cfIdList, CalculatedFieldState state, UUID tbMsgId, TbMsgType tbMsgType, TbCallback callback) { - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); - if (state.isReady() && ctx.isInitialized()) { - try { + try { + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(tenantId, ctx.getCfId(), entityId); + if (state.isReady() && ctx.isInitialized()) { CalculatedFieldResult calculationResult = state.performCalculation(ctx).get(5, TimeUnit.SECONDS); state.checkStateSize(ctxId, ctx.getMaxStateSizeInKBytes()); cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResultMap()), null); } - } catch (Exception e) { - if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { - systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, null, e); - } + } else { + callback.onSuccess(); // State was updated but no calculation performed; } - } else { - callback.onSuccess(); // State was updated but no calculation performed; - } - try { cfStateService.persistState(ctxId, state, callback); } catch (Exception e) { if (DebugModeUtil.isDebugFailuresAvailable(ctx.getCalculatedField())) { diff --git a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java index 6248ac1536..50ab512cb9 100644 --- a/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java +++ b/application/src/main/java/org/thingsboard/server/exception/CalculatedFieldStateException.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.exception; -public class CalculatedFieldStateException extends Exception { +public class CalculatedFieldStateException extends RuntimeException { public CalculatedFieldStateException(String message) { super(message); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index 0834b5583e..cdad1145e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -25,6 +25,7 @@ import org.thingsboard.script.api.tbel.TbelCfTsDoubleVal; import org.thingsboard.script.api.tbel.TbelCfTsRollingArg; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.exception.CalculatedFieldStateException; import java.util.ArrayList; import java.util.List; @@ -85,7 +86,7 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } @Override - public boolean updateEntry(ArgumentEntry entry) { + public boolean updateEntry(ArgumentEntry entry) throws CalculatedFieldStateException { if (entry instanceof TsRollingArgumentEntry tsRollingEntry) { updateTsRollingEntry(tsRollingEntry); } else if (entry instanceof SingleValueArgumentEntry singleValueEntry) { @@ -107,15 +108,18 @@ public class TsRollingArgumentEntry implements ArgumentEntry { } private void addTsRecord(Long ts, KvEntry value) { - switch (value.getDataType()) { - case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); - case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); - case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); - case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); - //TODO: try catch + try { + switch (value.getDataType()) { + case LONG -> value.getLongValue().ifPresent(aLong -> tsRecords.put(ts, aLong.doubleValue())); + case DOUBLE -> value.getDoubleValue().ifPresent(aDouble -> tsRecords.put(ts, aDouble)); + case BOOLEAN -> value.getBooleanValue().ifPresent(aBoolean -> tsRecords.put(ts, aBoolean ? 1.0 : 0.0)); + case STRING -> value.getStrValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + case JSON -> value.getJsonValue().ifPresent(aString -> tsRecords.put(ts, Double.parseDouble(aString))); + } + cleanupExpiredRecords(); + } catch (Exception e) { + throw new IllegalArgumentException("Time series rolling arguments supports only numeric values."); } - cleanupExpiredRecords(); } private void addTsRecord(Long ts, double value) { diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 91aad7b1cd..f544f463fa 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -40,8 +40,10 @@ import org.thingsboard.server.controller.CalculatedFieldControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; @DaoSqlTest public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { @@ -84,20 +86,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("77.0"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF output -> perform calculation with updated output Output savedOutput = savedCalculatedField.getConfiguration().getOutput(); @@ -106,32 +110,35 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedOutput.setName("temperatureF"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("86.0"); + }); // update CF argument -> perform calculation with new argument Argument savedArgument = savedCalculatedField.getConfiguration().getArguments().get("T"); savedArgument.setRefEntityKey(new ReferencedEntityKey("deviceTemperature", ArgumentType.ATTRIBUTE, AttributeScope.SERVER_SCOPE)); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); // update CF expression -> perform calculation with new expression savedCalculatedField.getConfiguration().setExpression("1.8 * T + 32"); savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); - Thread.sleep(300); - - temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); - assertThat(temperatureF).isNotNull(); - assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); + assertThat(temperatureF).isNotNull(); + assertThat(temperatureF.get(0).get("value").asText()).isEqualTo("104.0"); + }); } @Test @@ -164,20 +171,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> state is not ready -> no calculation performed CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> perform calculation doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -211,20 +220,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> perform initial calculation with default value CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("53.6"); + }); // update telemetry -> recalculate state doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").asText()).isEqualTo("86.0"); + }); } @Test @@ -275,93 +286,100 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF and perform initial calculation doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - // result of asset 1 - ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("51.0"); - // result of asset 2 - ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("52.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); - Thread.sleep(300); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("36.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 1 telemetry -> recalculate state only for asset 1 doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 (no changes) - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + // result of asset 2 (no changes) + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("37.0"); + }); // update asset 2 telemetry -> recalculate state only for asset 2 doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); - Thread.sleep(300); - - // result of asset 1 (no changes) - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 (no changes) + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("40.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("30.0"); + }); // add new entity to profile -> calculate state for new entity Asset asset3 = createAsset("Test asset 3", assetProfile.getId()); doPost("/api/plugins/telemetry/ASSET/" + asset3.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":13}")); - Thread.sleep(300); - - // result of asset 3 - ArrayNode z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + Asset finalAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("38.0"); + }); // update device telemetry -> recalculate state for all assets doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("35.0"); - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("25.0"); - // result of asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + // result of asset 3 + ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); // update profile for asset 3 -> delete state for asset 3 AssetProfile newAssetProfile = doPost("/api/assetProfile", createAssetProfile("New Asset Profile"), AssetProfile.class); @@ -371,22 +389,24 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // update device telemetry -> recalculate state for asset 1 and asset 2 doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":15}")); - Thread.sleep(300); - - // result of asset 1 - z1 = getServerAttributes(asset1.getId(), "z"); - assertThat(z1).isNotNull(); - assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); - - // result of asset 2 - z2 = getServerAttributes(asset2.getId(), "z"); - assertThat(z2).isNotNull(); - assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); - - // no changes for asset 3 - z3 = getServerAttributes(asset3.getId(), "z"); - assertThat(z3).isNotNull(); - assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + Asset updatedAsset3 = asset3; + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + // result of asset 1 + ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); + assertThat(z1).isNotNull(); + assertThat(z1.get(0).get("value").asText()).isEqualTo("30.0"); + + // result of asset 2 + ArrayNode z2 = getServerAttributes(asset2.getId(), "z"); + assertThat(z2).isNotNull(); + assertThat(z2.get(0).get("value").asText()).isEqualTo("20.0"); + + // no changes for asset 3 + ArrayNode z3 = getServerAttributes(updatedAsset3.getId(), "z"); + assertThat(z3).isNotNull(); + assertThat(z3.get(0).get("value").asText()).isEqualTo("33.0"); + }); } @Test @@ -421,20 +441,22 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes // create CF -> ctx is not initialized -> no calculation perform CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); - Thread.sleep(300); - - ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); // update telemetry -> ctx is not initialized -> no calculation perform doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); - Thread.sleep(300); - - fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); - assertThat(fahrenheitTemp).isNotNull(); - assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + await().atMost(5, TimeUnit.SECONDS) + .untilAsserted(() -> { + ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); + assertThat(fahrenheitTemp).isNotNull(); + assertThat(fahrenheitTemp.get("fahrenheitTemp").get(0).get("value").isNull()).isTrue(); + }); } private ObjectNode getLatestTelemetry(EntityId entityId, String... keys) throws Exception { From f342bd989ef891be8a56e8ea35ba2ca4c0e024e1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 13 Feb 2025 15:31:09 +0200 Subject: [PATCH 204/438] Minor improvements --- .../auth/rest/RestAuthenticationProvider.java | 1 - .../queue/discovery/HashPartitionService.java | 30 ++++++------------- .../server/queue/kafka/KafkaTbQueueMsg.java | 2 +- .../server/queue/util/AfterStartUp.java | 8 ++--- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java index 56215b82ac..b9fe54deec 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAuthenticationProvider.java @@ -106,7 +106,6 @@ public class RestAuthenticationProvider implements AuthenticationProvider { if (twoFactorAuthService.isTwoFaEnabled(securityUser.getTenantId(), securityUser.getId())) { return new MfaAuthenticationToken(securityUser); } else { - systemSecurityService.logLoginAction(securityUser, authentication.getDetails(), ActionType.LOGIN, null); } } else { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 249dfa859f..751e235adc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -228,7 +228,7 @@ public class HashPartitionService implements PartitionService { }); if (serviceInfoProvider.isService(ServiceType.TB_RULE_ENGINE)) { publishPartitionChangeEvent(ServiceType.TB_RULE_ENGINE, queueKeys.stream() - .collect(Collectors.toMap(k -> k, k -> Collections.emptySet())), Collections.emptyMap()); + .collect(Collectors.toMap(k -> k, k -> Collections.emptySet()))); } } @@ -408,7 +408,6 @@ public class HashPartitionService implements PartitionService { myPartitions = newPartitions; Map> changedPartitionsMap = new HashMap<>(); - Map> oldPartitionsMap = new HashMap<>(); Set removed = new HashSet<>(); oldPartitions.forEach((queueKey, partitions) -> { @@ -429,16 +428,16 @@ public class HashPartitionService implements PartitionService { myPartitions.forEach((queueKey, partitions) -> { if (!partitions.equals(oldPartitions.get(queueKey))) { - changedPartitionsMap.put(queueKey, toTpiList(queueKey, partitions)); - oldPartitionsMap.put(queueKey, toTpiList(queueKey, oldPartitions.get(queueKey))); + Set tpiList = partitions.stream() + .map(partition -> buildTopicPartitionInfo(queueKey, partition)) + .collect(Collectors.toSet()); + changedPartitionsMap.put(queueKey, tpiList); } }); if (!changedPartitionsMap.isEmpty()) { changedPartitionsMap.entrySet().stream() .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) - .forEach((serviceType, partitionsMap) -> { - publishPartitionChangeEvent(serviceType, partitionsMap, oldPartitionsMap); - }); + .forEach(this::publishPartitionChangeEvent); } if (currentOtherServices == null) { @@ -470,15 +469,13 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } - private void publishPartitionChangeEvent(ServiceType serviceType, - Map> newPartitions, - Map> oldPartitions) { - log.info("Partitions changed: {}", System.lineSeparator() + newPartitions.entrySet().stream() + private void publishPartitionChangeEvent(ServiceType serviceType, Map> partitionsMap) { + log.info("Partitions changed: {}", System.lineSeparator() + partitionsMap.entrySet().stream() .map(entry -> "[" + entry.getKey() + "] - [" + entry.getValue().stream() .map(tpi -> tpi.getPartition().orElse(-1).toString()).sorted() .collect(Collectors.joining(", ")) + "]") .collect(Collectors.joining(System.lineSeparator()))); - PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, newPartitions); + PartitionChangeEvent event = new PartitionChangeEvent(this, serviceType, partitionsMap); try { applicationEventPublisher.publishEvent(event); } catch (Exception e) { @@ -486,15 +483,6 @@ public class HashPartitionService implements PartitionService { } } - private Set toTpiList(QueueKey queueKey, List partitions) { - if (partitions == null) { - return Collections.emptySet(); - } - return partitions.stream() - .map(partition -> buildTopicPartitionInfo(queueKey, partition)) - .collect(Collectors.toSet()); - } - @Override public Set getAllServiceIds(ServiceType serviceType) { return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java index fe2c8b8a90..302f36ff1a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/KafkaTbQueueMsg.java @@ -31,7 +31,7 @@ public class KafkaTbQueueMsg implements TbQueueMsg { private final byte[] data; public KafkaTbQueueMsg(ConsumerRecord record) { - if (record.key().length() == UUID_LENGTH) { + if (record.key().length() <= UUID_LENGTH) { this.key = UUID.fromString(record.key()); } else { this.key = UUID.randomUUID(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index a24f0663b5..6943fa6753 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2024 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From fb0b80a35321710f18a496e46f58f2f8a02699e8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:02:46 +0200 Subject: [PATCH 205/438] Calculated field adjustments --- .../calculated-fields-table-config.ts | 2 +- .../calculated-field-dialog.component.html | 24 ++++++++++++------- .../entity/entities-table.component.ts | 2 +- .../components/event/event-table-config.ts | 13 +++++----- .../entity/entities-table-config.models.ts | 6 ++++- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index ce47292397..f1e549d75f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -280,7 +280,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig { if (openCalculatedFieldEdit) { - this.editCalculatedField({...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) + this.editCalculatedField({ entityId: this.entityId, ...calculatedField, configuration: {...calculatedField.configuration, expression } }, true) } }), ); diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 6ff0977178..8bd3b5db3f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -100,12 +100,22 @@ [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" helpId="calculated-field/expression_fn" - /> + > + +

@@ -122,18 +132,16 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute) { + @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { {{ 'calculated-fields.attribute-scope' | translate }} {{ 'calculated-fields.server-attributes' | translate }} - @if (data.entityId.entityType === EntityType.DEVICE) { - - {{ 'calculated-fields.shared-attributes' | translate }} - - } + + {{ 'calculated-fields.shared-attributes' | translate }} + } diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index 4f6f34fcce..e472ad5389 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -689,7 +689,7 @@ export class EntitiesTableComponent extends PageComponent implements IEntitiesTa } cellTooltip(entity: BaseData, column: EntityColumn>, row: number) { - if (column instanceof EntityTableColumn) { + if (column instanceof EntityTableColumn || column instanceof EntityLinkTableColumn) { const col = this.entitiesTableConfig.columns.indexOf(column); const index = row * this.entitiesTableConfig.columns.length + col; let res = this.cellTooltipCache[index]; diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index b32c5f0050..a2ccaa05da 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -18,6 +18,7 @@ import { CellActionDescriptorType, DateEntityTableColumn, EntityActionTableColumn, + EntityLinkTableColumn, EntityTableColumn, EntityTableConfig } from '@home/models/entity/entities-table-config.models'; @@ -29,7 +30,7 @@ import { MatDialog } from '@angular/material/dialog'; import { EntityId } from '@shared/models/id/entity-id'; import { EventService } from '@app/core/http/event.service'; import { EventTableHeaderComponent } from '@home/components/event/event-table-header.component'; -import { EntityTypeResource } from '@shared/models/entity-type.models'; +import { EntityType, EntityTypeResource } from '@shared/models/entity-type.models'; import { fromEvent, Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { Direction } from '@shared/models/page/sort-order'; @@ -39,7 +40,7 @@ import { EventContentDialogComponent, EventContentDialogData } from '@home/components/event/event-content-dialog.component'; -import { isEqual, sortObjectKeys } from '@core/utils'; +import { getEntityDetailsPageURL, isEqual, sortObjectKeys } from '@core/utils'; import { DAY, historyInterval, MINUTE } from '@shared/models/time/time.models'; import { Overlay, OverlayConfig, OverlayRef } from '@angular/cdk/overlay'; import { ChangeDetectorRef, EventEmitter, Injector, StaticProvider, ViewContainerRef } from '@angular/core'; @@ -359,13 +360,13 @@ export class EventTableConfig extends EntityTableConfig { this.columns[0].width = '80px'; this.columns[1].width = '100px'; this.columns.push( - new EntityTableColumn('entityId', 'event.entity-id', '100px', + new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, + (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, - false, + (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), icon: 'content_paste', @@ -384,7 +385,7 @@ export class EventTableConfig extends EntityTableConfig { () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), - () => undefined, + (entity) => entity.body.msgId, false, { name: this.translate.instant('event.copy-message-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index dbdcdcd61e..6c70a11326 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,7 +121,11 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public sortable: boolean = true) { + public cellStyleFunction: CellStyleFunction = () => ({}), + public sortable: boolean = true, + public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), + public cellTooltipFunction: CellTooltipFunction = () => undefined, + public actionCell: CellActionDescriptor = null) { super('link', key, title, width, sortable); } } From 5aacd04e151bb7184dc024c0a6e79c47998b924e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:11:11 +0200 Subject: [PATCH 206/438] Moved calculated fields in device profile tabs --- .../dialog/calculated-field-dialog.component.html | 3 ++- .../device-profile/device-profile-tabs.component.html | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 8bd3b5db3f..2f707c0a2f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -132,7 +132,8 @@ } - @if (outputFormGroup.get('type').value === OutputType.Attribute && data.entityId.entityType === EntityType.DEVICE) { + @if (outputFormGroup.get('type').value === OutputType.Attribute + && (data.entityId.entityType === EntityType.DEVICE || data.entityId.entityType === EntityType.DEVICE_PROFILE)) { {{ 'calculated-fields.attribute-scope' | translate }} diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index 37cfafa9aa..9a3c34c070 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -40,6 +40,10 @@
+ + + - - From 734064d8b1cc12fb4a222d9feae68b09b77b5816 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Thu, 13 Feb 2025 17:44:04 +0200 Subject: [PATCH 207/438] Switched link column config order --- .../src/app/modules/home/components/event/event-table-config.ts | 2 +- .../modules/home/models/entity/entities-table-config.models.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index a2ccaa05da..1e9717d5a8 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -363,9 +363,9 @@ export class EventTableConfig extends EntityTableConfig { new EntityLinkTableColumn('entityId', 'event.entity-id', '100px', (entity) => `${entity.body.entityId.substring(0, 8)}…`, (entity) => getEntityDetailsPageURL(entity.body.entityId, entity.body.entityType as EntityType), - () => ({padding: '0 12px 0 0'}), false, () => ({padding: '0 12px 0 0'}), + () => ({padding: '0 12px 0 0'}), (entity) => entity.body.entityId, { name: this.translate.instant('event.copy-entity-id'), diff --git a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts index 6c70a11326..49583e90dc 100644 --- a/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts +++ b/ui-ngx/src/app/modules/home/models/entity/entities-table-config.models.ts @@ -121,8 +121,8 @@ export class EntityLinkTableColumn> extends BaseEntity public width: string = '0px', public cellContentFunction: CellContentFunction = (entity, property) => entity[property] ? entity[property] : '', public entityURL: (entity) => string, - public cellStyleFunction: CellStyleFunction = () => ({}), public sortable: boolean = true, + public cellStyleFunction: CellStyleFunction = () => ({}), public headerCellStyleFunction: HeaderCellStyleFunction = () => ({}), public cellTooltipFunction: CellTooltipFunction = () => undefined, public actionCell: CellActionDescriptor = null) { From bb7ea08e59f1c5300229eb92451532e37cb63751 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Feb 2025 12:08:05 +0200 Subject: [PATCH 208/438] EDQS: refactoring, OOM handling, healthcheck --- .../TbRuleEngineQueueConsumerManager.java | 2 +- application/src/main/resources/logback.xml | 1 + .../EdqsEntityQueryControllerTest.java | 3 +- .../server/common/data/ObjectType.java | 8 +- common/edqs/pom.xml | 4 + .../server/edqs/processor/EdqsProcessor.java | 30 +++++-- .../server/edqs/processor/EdqsProducer.java | 2 +- .../server/edqs/repo/TenantRepo.java | 16 ++-- .../server/edqs/state/EdqsController.java | 40 +++++++++ .../server/edqs/state/EdqsStateService.java | 2 + .../edqs/state/KafkaEdqsStateService.java | 85 +++++++++++-------- .../edqs/state/LocalEdqsStateService.java | 12 ++- .../server/edqs/stats/EdqsStatsService.java | 6 +- .../consumer/MainQueueConsumerManager.java | 52 +++++++----- .../common/consumer/TbQueueConsumerTask.java | 2 +- .../kafka/TbKafkaConsumerStatsService.java | 13 +-- 16 files changed, 179 insertions(+), 99 deletions(-) create mode 100644 common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 3636eb05af..1f421974be 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -72,7 +72,7 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager + diff --git a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java index 9bb0bbe30a..1add0dae37 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdqsEntityQueryControllerTest.java @@ -28,7 +28,6 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; import java.util.concurrent.TimeUnit; -import java.util.function.BiPredicate; import static org.awaitility.Awaitility.await; @@ -45,7 +44,7 @@ public class EdqsEntityQueryControllerTest extends EntityQueryControllerTest { @Autowired private EdqsService edqsService; - @MockBean + @MockBean // so that we don't do backup for tests private EdqsRocksDb edqsRocksDb; @Before diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index bc5ec58213..86252fb7f8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -15,9 +15,8 @@ */ package org.thingsboard.server.common.data; -import java.util.Arrays; import java.util.EnumSet; -import java.util.HashSet; +import java.util.List; import java.util.Set; public enum ObjectType { @@ -68,12 +67,13 @@ public enum ObjectType { TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); - public static final Set edqsTypes = new HashSet<>(edqsTenantTypes); + public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); + public static final Set unversionedTypes = EnumSet.of(QUEUE_STATS); static { - edqsTypes.addAll(Arrays.asList(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); + edqsTypes.addAll(List.of(RELATION, ATTRIBUTE_KV, LATEST_TS_KV)); } public EntityType toEntityType() { diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index e58c7c97a0..2abd1286b8 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -68,6 +68,10 @@ org.thingsboard.common queue + + org.springframework.boot + spring-boot-starter-web + org.apache.kafka kafka-clients diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 6fe09f77d1..a90a029bc4 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; @@ -72,6 +74,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; import java.util.stream.Collectors; @EdqsComponent @@ -85,8 +88,8 @@ public class EdqsProcessor implements TbQueueHandler, private final EdqRepository repository; private final EdqsConfig config; private final EdqsPartitionService partitionService; - @Autowired - @Lazy + private final ConfigurableApplicationContext applicationContext; + @Autowired @Lazy private EdqsStateService stateService; private MainQueueConsumerManager, QueueConfig> eventsConsumer; @@ -96,17 +99,30 @@ public class EdqsProcessor implements TbQueueHandler, private ExecutorService mgmtExecutor; private ScheduledExecutorService scheduler; private ListeningExecutorService requestExecutor; + private ExecutorService repartitionExecutor; private final VersionsStore versionsStore = new VersionsStore(); private final AtomicInteger counter = new AtomicInteger(); // FIXME: TMP + @Getter + private Consumer errorHandler; + @PostConstruct private void init() { consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-scheduler"); requestExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(12, "edqs-requests")); + repartitionExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-repartition")); + errorHandler = error -> { + if (error instanceof OutOfMemoryError) { + log.error("OOM detected, shutting down"); + repository.clear(); + Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edqs-shutdown")) + .execute(applicationContext::close); + } + }; eventsConsumer = MainQueueConsumerManager., QueueConfig>builder() .queueKey(new QueueKey(ServiceType.EDQS, EdqsQueue.EVENTS.getTopic())) @@ -117,7 +133,7 @@ public class EdqsProcessor implements TbQueueHandler, ToEdqsMsg msg = queueMsg.getValue(); log.trace("Processing message: {}", msg); process(msg, EdqsQueue.EVENTS); - } catch (Throwable t) { + } catch (Exception t) { log.error("Failed to process message: {}", queueMsg, t); } } @@ -127,6 +143,7 @@ public class EdqsProcessor implements TbQueueHandler, .consumerExecutor(consumersExecutor) .taskExecutor(mgmtExecutor) .scheduler(scheduler) + .uncaughtErrorHandler(errorHandler) .build(); responseTemplate = queueFactory.createEdqsResponseTemplate(); } @@ -141,7 +158,7 @@ public class EdqsProcessor implements TbQueueHandler, if (event.getServiceType() != ServiceType.EDQS) { return; } - consumersExecutor.submit(() -> { + repartitionExecutor.submit(() -> { // todo: maybe cancel the task if new event comes try { Set newPartitions = event.getNewPartitions().get(new QueueKey(ServiceType.EDQS)); Set partitions = newPartitions.stream() @@ -220,8 +237,8 @@ public class EdqsProcessor implements TbQueueHandler, if (!versionsStore.isNew(key, version)) { return; } - } else { - log.warn("[{}] {} doesn't have version: {}", tenantId, objectType, edqsMsg); + } else if (!ObjectType.unversionedTypes.contains(objectType)) { + log.warn("[{}] {} {} doesn't have version", tenantId, objectType, key); } if (queue != EdqsQueue.STATE) { stateService.save(tenantId, objectType, key, eventType, edqsMsg); @@ -272,6 +289,7 @@ public class EdqsProcessor implements TbQueueHandler, mgmtExecutor.shutdownNow(); scheduler.shutdownNow(); requestExecutor.shutdownNow(); + repartitionExecutor.shutdownNow(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java index b21184110f..9b1e925367 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProducer.java @@ -53,7 +53,7 @@ public class EdqsProducer { TbQueueCallback callback = new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { - log.debug("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels + log.trace("[{}][{}][{}] Published msg to {}: {}", tenantId, type, key, topic, msg); // fixme log levels } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 68e1c42589..822061e918 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -153,7 +153,7 @@ public class TenantRepo { EntityData to = getOrCreate(entity.getTo()); boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); } } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { @@ -172,7 +172,7 @@ public class TenantRepo { if (relationsRepo != null) { boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); } } } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { @@ -223,7 +223,7 @@ public class TenantRepo { EntityData removed = getEntityMap(entityType).remove(entityId); if (removed != null) { getEntitySet(entityType).remove(removed); - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); UUID customerId = removed.getCustomerId(); if (customerId != null) { CustomerData customerData = (CustomerData) getEntityMap(EntityType.CUSTOMER).get(customerId); @@ -244,7 +244,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(attributeKv.getKey()); boolean added = entityData.putAttr(keyId, attributeKv.getScope(), toDataPoint(attributeKv.getLastUpdateTs(), value)); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); } } } @@ -254,7 +254,7 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); } } } @@ -266,7 +266,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(latestTsKv.getKey()); boolean added = entityData.putTs(keyId, toDataPoint(latestTsKv.getTs(), value)); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); } } } @@ -276,7 +276,7 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); } } } @@ -331,7 +331,7 @@ public class TenantRepo { log.debug("[{}] Adding {} {}", tenantId, entityType, id); EntityData entityData = constructEntityData(entityType, entityId); getEntitySet(entityType).add(entityData); - edqsStatsService.ifPresent(statService -> statService.reportTenantEdqsObject(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); return entityData; }); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java new file mode 100644 index 0000000000..721675245a --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequiredArgsConstructor +@RequestMapping("/api/edqs") +public class EdqsController { + + private final EdqsStateService edqsStateService; + + @GetMapping("/ready") + public ResponseEntity isReady() { + if (edqsStateService.isReady()) { + return ResponseEntity.ok().build(); + } else { + return ResponseEntity.badRequest().build(); + } + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java index 8866a1c771..06d080ac64 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsStateService.java @@ -29,4 +29,6 @@ public interface EdqsStateService { void save(TenantId tenantId, ObjectType type, String key, EdqsEventType eventType, ToEdqsMsg msg); + boolean isReady(); + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index 84f3cfd22f..cfa831f720 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -42,10 +42,10 @@ import org.thingsboard.server.queue.edqs.EdqsConfig; import org.thingsboard.server.queue.edqs.EdqsQueue; import org.thingsboard.server.queue.edqs.EdqsQueueFactory; import org.thingsboard.server.queue.edqs.KafkaEdqsComponent; -import org.thingsboard.server.queue.util.AfterStartUp; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -66,6 +66,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private QueueConsumerManager> eventsConsumer; private EdqsProducer stateProducer; + private boolean initialRestoreDone; + private ExecutorService consumersExecutor; private ExecutorService mgmtExecutor; private ScheduledExecutorService scheduler; @@ -76,7 +78,7 @@ public class KafkaEdqsStateService implements EdqsStateService { @PostConstruct private void init() { - consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-backup-consumer")); + consumersExecutor = Executors.newCachedThreadPool(ThingsBoardThreadFactory.forName("edqs-consumer")); mgmtExecutor = ThingsBoardExecutors.newWorkStealingPool(4, "edqs-backup-consumer-mgmt"); scheduler = ThingsBoardExecutors.newSingleThreadScheduledExecutor("edqs-backup-scheduler"); @@ -92,8 +94,8 @@ public class KafkaEdqsStateService implements EdqsStateService { if (stateReadCount.incrementAndGet() % 100000 == 0) { log.info("[state] Processed {} msgs", stateReadCount.get()); } - } catch (Throwable t) { - log.error("Failed to process message: {}", queueMsg, t); + } catch (Exception e) { + log.error("Failed to process message: {}", queueMsg, e); // TODO: do something about the error - e.g. reprocess } } consumer.commit(); @@ -102,39 +104,48 @@ public class KafkaEdqsStateService implements EdqsStateService { .consumerExecutor(consumersExecutor) .taskExecutor(mgmtExecutor) .scheduler(scheduler) + .uncaughtErrorHandler(edqsProcessor.getErrorHandler()) .build(); - eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start + ExecutorService backupExecutor = ThingsBoardExecutors.newLimitedTasksExecutor(12, 1000, "events-to-backup-executor"); + eventsConsumer = QueueConsumerManager.>builder() // FIXME Slavik writes to the state while we read it, slows down the start. maybe start backup consumer after restore is finished .name("edqs-events-to-backup-consumer") .pollInterval(config.getPollInterval()) .msgPackProcessor((msgs, consumer) -> { + CountDownLatch resultLatch = new CountDownLatch(msgs.size()); for (TbProtoQueueMsg queueMsg : msgs) { - try { - ToEdqsMsg msg = queueMsg.getValue(); - log.trace("Processing message: {}", msg); - - if (msg.hasEventMsg()) { - EdqsEventMsg eventMsg = msg.getEventMsg(); - String key = eventMsg.getKey(); - if (eventsReadCount.incrementAndGet() % 100000 == 0) { - log.info("[events-to-backup] Processed {} msgs", eventsReadCount.get()); - } - if (eventMsg.hasVersion()) { - if (!versionsStore.isNew(key, eventMsg.getVersion())) { - return; + backupExecutor.submit(() -> { + try { + ToEdqsMsg msg = queueMsg.getValue(); + log.trace("Processing message: {}", msg); + + if (msg.hasEventMsg()) { + EdqsEventMsg eventMsg = msg.getEventMsg(); + String key = eventMsg.getKey(); + int count = eventsReadCount.incrementAndGet(); + if (count % 100000 == 0) { + log.info("[events-to-backup] Processed {} msgs", count); + } + if (eventMsg.hasVersion()) { + if (!versionsStore.isNew(key, eventMsg.getVersion())) { + return; + } } - } - TenantId tenantId = getTenantId(msg); - ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); - EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); - log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); - stateProducer.send(tenantId, objectType, key, msg); + TenantId tenantId = getTenantId(msg); + ObjectType objectType = ObjectType.valueOf(eventMsg.getObjectType()); + EdqsEventType eventType = EdqsEventType.valueOf(eventMsg.getEventType()); + log.debug("[{}] Saving to backup [{}] [{}] [{}]", tenantId, objectType, eventType, key); + stateProducer.send(tenantId, objectType, key, msg); + } + } catch (Throwable t) { + log.error("Failed to process message: {}", queueMsg, t); + } finally { + resultLatch.countDown(); } - } catch (Throwable t) { - log.error("Failed to process message: {}", queueMsg, t); - } + }); } + resultLatch.await(); consumer.commit(); }) .consumerCreator(() -> queueFactory.createEdqsMsgConsumer(EdqsQueue.EVENTS, "events-to-backup-consumer-group")) // shared by all instances consumer group @@ -149,22 +160,21 @@ public class KafkaEdqsStateService implements EdqsStateService { .build(); } - @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) - public void afterStartUp() { - eventsConsumer.subscribe(); - eventsConsumer.launch(); - } - @Override public void restore(Set partitions) { stateReadCount.set(0); //TODO Slavik: do not support remote mode in monolith setup long startTs = System.currentTimeMillis(); log.info("Restore started for partitions {}", partitions.stream().map(tpi -> tpi.getPartition().orElse(-1)).sorted().toList()); - stateConsumer.doUpdate(partitions); // calling blocking doUpdate instead of update stateConsumer.awaitStop(0); // consumers should stop on their own because EdqsQueue.STATE.stopWhenRead is true, we just need to wait - log.info("Restore finished in {} ms. Processed {} msgs", (System.currentTimeMillis() - startTs), stateReadCount.get()); + + if (!initialRestoreDone) { + initialRestoreDone = true; + + eventsConsumer.subscribe(); + eventsConsumer.launch(); + } } @Override @@ -172,6 +182,11 @@ public class KafkaEdqsStateService implements EdqsStateService { // do nothing here, backup is done by events consumer } + @Override + public boolean isReady() { + return initialRestoreDone; + } + private TenantId getTenantId(ToEdqsMsg edqsMsg) { return TenantId.fromUUID(new UUID(edqsMsg.getTenantIdMSB(), edqsMsg.getTenantIdLSB())); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java index 8d2d563d90..6a61488182 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/LocalEdqsStateService.java @@ -43,13 +43,11 @@ public class LocalEdqsStateService implements EdqsStateService { @Autowired private EdqsRocksDb db; - private Set partitions; + private boolean restoreDone; @Override public void restore(Set partitions) { - if (this.partitions == null) { - this.partitions = partitions; - } else { + if (restoreDone) { return; } @@ -62,6 +60,7 @@ public class LocalEdqsStateService implements EdqsStateService { log.error("[{}] Failed to restore value", key, e); } }); + restoreDone = true; } @Override @@ -78,4 +77,9 @@ public class LocalEdqsStateService implements EdqsStateService { } } + @Override + public boolean isReady() { + return restoreDone; + } + } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java index 2ca036d23d..9619d086ad 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -52,9 +52,9 @@ public class EdqsStatsService { log.info("EDQS Stats: {}", values); } - public void reportTenantEdqsObject(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { + public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) - .reportEdqsObject(objectType, eventType); + .reportEvent(objectType, eventType); } @Getter @@ -78,7 +78,7 @@ public class EdqsStatsService { .collect(Collectors.joining(", ")); } - public void reportEdqsObject(ObjectType objectType, EdqsEventType eventType) { + public void reportEvent(ObjectType objectType, EdqsEventType eventType) { AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); if (eventType == EdqsEventType.UPDATED){ objectCounter.incrementAndGet(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index 2f44a9c4ae..9d45b168ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -39,6 +39,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.stream.Collectors; @Slf4j @@ -53,6 +54,7 @@ public class MainQueueConsumerManager uncaughtErrorHandler; private final java.util.Queue tasks = new ConcurrentLinkedQueue<>(); private final ReentrantLock lock = new ReentrantLock(); @@ -68,7 +70,8 @@ public class MainQueueConsumerManager> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, - ExecutorService taskExecutor) { + ExecutorService taskExecutor, + Consumer uncaughtErrorHandler) { this.queueKey = queueKey; this.config = config; this.msgPackProcessor = msgPackProcessor; @@ -76,6 +79,7 @@ public class MainQueueConsumerManager consumerLoop = consumerExecutor.submit(() -> { ThingsBoardThreadFactory.updateCurrentThreadName(consumerTask.getKey().toString()); - try { - consumerLoop(consumerTask.getConsumer()); - } catch (Throwable e) { - log.error("Failure in consumer loop", e); - } + consumerLoop(consumerTask.getConsumer()); log.info("[{}] Consumer stopped", consumerTask.getKey()); }); consumerTask.setTask(consumerLoop); } private void consumerLoop(TbQueueConsumer consumer) { - while (!stopped && !consumer.isStopped()) { - try { - List msgs = consumer.poll(config.getPollInterval()); - if (msgs.isEmpty()) { - continue; - } - processMsgs(msgs, consumer, config); - } catch (Exception e) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", e); - try { - Thread.sleep(config.getPollInterval()); - } catch (InterruptedException e2) { - log.trace("Failed to wait until the server has capacity to handle new requests", e2); + try { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(config.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + processMsgs(msgs, consumer, config); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(config.getPollInterval()); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new requests", e2); + } } } } - } - if (consumer.isStopped()) { + if (consumer.isStopped()) { + consumer.unsubscribe(); + } + } catch (Throwable t) { + log.error("Failure in consumer loop", t); + if (uncaughtErrorHandler != null) { + uncaughtErrorHandler.accept(t); + } consumer.unsubscribe(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java index 708e24fdac..4ed0ffa497 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/TbQueueConsumerTask.java @@ -84,7 +84,7 @@ public class TbQueueConsumerTask { } log.trace("[{}] Awaited finish", key); } catch (Exception e) { - log.warn("[{}] Failed to await for consumer to stop", key, e); + log.warn("[{}] Failed to await for consumer to stop (timeout {} sec)", key, timeoutSec, e); } task = null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java index 0367a2be78..ade9204572 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerStatsService.java @@ -26,15 +26,10 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.queue.discovery.PartitionService; import java.time.Duration; import java.util.ArrayList; @@ -56,10 +51,6 @@ public class TbKafkaConsumerStatsService { private final TbKafkaSettings kafkaSettings; private final TbKafkaConsumerStatisticConfig statsConfig; - @Lazy - @Autowired - private PartitionService partitionService; - private Consumer consumer; private ScheduledExecutorService statsPrintScheduler; @@ -111,9 +102,7 @@ public class TbKafkaConsumerStatsService { } private boolean isStatsPrintRequired() { - boolean isMyRuleEnginePartition = partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - boolean isMyCorePartition = partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); - return log.isInfoEnabled() && (isMyRuleEnginePartition || isMyCorePartition); + return log.isInfoEnabled(); } private List getTopicsStatsWithLag(Map groupOffsets, Map endOffsets) { From 13fbcf4ba0fd5de64d6ec49fa9842973795384c5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Feb 2025 12:31:36 +0200 Subject: [PATCH 209/438] Fix init for readiness probe controller --- .../java/org/thingsboard/server/edqs/state/EdqsController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java index 721675245a..06d5f89aa9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/EdqsController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.state; import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -23,6 +24,7 @@ import org.springframework.web.bind.annotation.RestController; @RestController @RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='edqs'") @RequestMapping("/api/edqs") public class EdqsController { From 52a6540e5a7346e01239ebf7a63cf1c9f15d23df Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 14 Feb 2025 15:12:00 +0200 Subject: [PATCH 210/438] added docker image for edqs --- edqs/src/main/conf/edqs.conf | 22 ++++ edqs/src/main/conf/logback.xml | 49 ++++++++ msa/edqs/docker/Dockerfile | 31 +++++ msa/edqs/docker/start-tb-edqs.sh | 31 +++++ msa/edqs/pom.xml | 190 +++++++++++++++++++++++++++++++ msa/pom.xml | 1 + 6 files changed, 324 insertions(+) create mode 100644 edqs/src/main/conf/edqs.conf create mode 100644 edqs/src/main/conf/logback.xml create mode 100644 msa/edqs/docker/Dockerfile create mode 100755 msa/edqs/docker/start-tb-edqs.sh create mode 100644 msa/edqs/pom.xml diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf new file mode 100644 index 0000000000..ae880ef7e8 --- /dev/null +++ b/edqs/src/main/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml new file mode 100644 index 0000000000..4ff57d48ab --- /dev/null +++ b/edqs/src/main/conf/logback.xml @@ -0,0 +1,49 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/msa/edqs/docker/Dockerfile b/msa/edqs/docker/Dockerfile new file mode 100644 index 0000000000..e634029bbf --- /dev/null +++ b/msa/edqs/docker/Dockerfile @@ -0,0 +1,31 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk17:bookworm-slim + +COPY start-tb-edqs.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-edqs.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chown -R ${pkg.user}:${pkg.user} /tmp && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-edqs.sh"] \ No newline at end of file diff --git a/msa/edqs/docker/start-tb-edqs.sh b/msa/edqs/docker/start-tb-edqs.sh new file mode 100755 index 0000000000..bdd1ee48d3 --- /dev/null +++ b/msa/edqs/docker/start-tb-edqs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER=${pkg.installFolder}/conf +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.edqs.ThingsboardEdqsApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=$CONF_FOLDER/logback.xml \ + org.springframework.boot.loader.launch.PropertiesLauncher diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml new file mode 100644 index 0000000000..4874c06e4b --- /dev/null +++ b/msa/edqs/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + msa + + org.thingsboard.msa + edqs + pom + + ThingsBoard Entity Data Query Microservice + https://thingsboard.io + ThingsBoard Entity Data Query Microservice + + + UTF-8 + ${basedir}/../.. + edqs + edqs + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard + edqs + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-edqs + package + + copy + + + + + org.thingsboard + edqs + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/pom.xml b/msa/pom.xml index ff394295f4..259de5fb93 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -48,6 +48,7 @@ transport js-executor monitoring + edqs From 53f3da153d03828174a1302ef0c68c451562b8b8 Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 15:28:28 +0200 Subject: [PATCH 211/438] Added debug events action from calculated field table --- .../calculated-fields/calculated-fields-table-config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index f1e549d75f..e7ad697262 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -118,6 +118,12 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, onAction: (event$, entity) => this.exportCalculatedField(event$, entity), }, + { + name: this.translate.instant('entity-view.events'), + icon: 'history', + isEnabled: () => true, + onAction: (_, entity) => this.openDebugDialog(entity), + }, { name: '', nameFunction: entity => this.getDebugConfigLabel(entity?.debugSettings), From 7911384f375f8e823afd51974a53315598673aa4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 14 Feb 2025 16:42:55 +0200 Subject: [PATCH 212/438] fixed tests, added error without stacktrace and param to system params --- .../server/actors/ActorSystemContext.java | 74 +++++++++++-------- ...alculatedFieldManagerMessageProcessor.java | 14 ++-- .../controller/SystemInfoController.java | 11 ++- .../cf/ctx/state/TsRollingArgumentEntry.java | 2 +- .../src/main/resources/thingsboard.yml | 6 ++ .../state/ScriptCalculatedFieldStateTest.java | 55 ++------------ .../state/SimpleCalculatedFieldStateTest.java | 10 +-- .../state/SingleValueArgumentEntryTest.java | 7 +- .../ctx/state/TsRollingArgumentEntryTest.java | 60 +++++++++++---- .../server/common/data/SystemParams.java | 1 + .../data/event/CalculatedFieldDebugEvent.java | 1 - 11 files changed, 125 insertions(+), 116 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b8344c6c0a..20f61115e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -43,7 +43,6 @@ import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cache.limits.RateLimitService; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.event.CalculatedFieldDebugEvent; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; @@ -624,6 +623,14 @@ public class ActorSystemContext { @Getter private String debugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + @Getter + private boolean calculatedFieldsDebugPerTenantEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + @Getter + private String calculatedFieldsDebugPerTenantLimitsConfiguration; + @Value("${actors.rpc.submit_strategy:BURST}") @Getter private String rpcSubmitStrategy; @@ -810,39 +817,42 @@ public class ActorSystemContext { } public void persistCalculatedFieldDebugEvent(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, Map arguments, UUID tbMsgId, TbMsgType tbMsgType, String result, Throwable error) { - if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, tenantId)) { - throw new TbRateLimitsException(EntityType.TENANT); - } - try { - CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() - .tenantId(tenantId) - .entityId(calculatedFieldId.getId()) - .serviceId(getServiceId()) - .calculatedFieldId(calculatedFieldId) - .eventEntity(entityId); - if (tbMsgId != null) { - eventBuilder.msgId(tbMsgId); - } - if (tbMsgType != null) { - eventBuilder.msgType(tbMsgType.name()); - } - if (arguments != null) { - eventBuilder.arguments(JacksonUtil.toString( - arguments.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) - )); - } - if (result != null) { - eventBuilder.result(result); - } - if (error != null) { - eventBuilder.error(toString(error)); + if (calculatedFieldsDebugPerTenantEnabled) { + if (!rateLimitService.checkRateLimit(LimitedApi.CALCULATED_FIELD_DEBUG_EVENTS, (Object) tenantId, calculatedFieldsDebugPerTenantLimitsConfiguration)) { + log.trace("[{}] Calculated field debug event limits exceeded!", tenantId); + throw new TbRateLimitsException("Failed to persist calculated field debug event due to rate limits!"); } + try { + CalculatedFieldDebugEvent.CalculatedFieldDebugEventBuilder eventBuilder = CalculatedFieldDebugEvent.builder() + .tenantId(tenantId) + .entityId(calculatedFieldId.getId()) + .serviceId(getServiceId()) + .calculatedFieldId(calculatedFieldId) + .eventEntity(entityId); + if (tbMsgId != null) { + eventBuilder.msgId(tbMsgId); + } + if (tbMsgType != null) { + eventBuilder.msgType(tbMsgType.name()); + } + if (arguments != null) { + eventBuilder.arguments(JacksonUtil.toString( + arguments.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toTbelCfArg())) + )); + } + if (result != null) { + eventBuilder.result(result); + } + if (error != null) { + eventBuilder.error(error.getMessage()); + } - ListenableFuture future = eventService.saveAsync(eventBuilder.build()); - Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); - } catch (IllegalArgumentException ex) { - log.warn("Failed to persist calculated field debug message", ex); + ListenableFuture future = eventService.saveAsync(eventBuilder.build()); + Futures.addCallback(future, CALCULATED_FIELD_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); + } catch (IllegalArgumentException ex) { + log.warn("Failed to persist calculated field debug message", ex); + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 898e339426..c3aa378524 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -262,6 +262,13 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); + try { + newCfCtx.init(); + } catch (Exception e) { + if (DebugModeUtil.isDebugAllAvailable(newCf)) { + systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); + } + } calculatedFields.put(newCf.getId(), newCfCtx); List oldCfList = entityIdCalculatedFields.get(newCf.getEntityId()); List newCfList = new ArrayList<>(oldCfList.size()); @@ -286,13 +293,6 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware // Alternative approach would be to use any list but avoid modifications to the list (change the complete map value instead) var stateChanges = newCfCtx.hasStateChanges(oldCfCtx); if (stateChanges || newCfCtx.hasOtherSignificantChanges(oldCfCtx)) { - try { - newCfCtx.init(); - } catch (Exception e) { - if (DebugModeUtil.isDebugAllAvailable(newCf)) { - systemContext.persistCalculatedFieldDebugEvent(newCf.getTenantId(), newCf.getId(), newCf.getEntityId(), null, null, null, null, e); - } - } initCf(newCfCtx, callback, stateChanges); } else { callback.onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java index 8b958df68b..d630c5a22a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java +++ b/application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.SystemParams; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.mobile.qrCodeSettings.QRCodeConfig; +import org.thingsboard.server.common.data.mobile.qrCodeSettings.QrCodeSettings; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -80,6 +80,12 @@ public class SystemInfoController extends BaseController { @Value("${actors.rule.chain.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") private String ruleChainDebugPerTenantLimitsConfiguration; + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.enabled:true}") + private boolean calculatedFieldDebugPerTenantLimitsEnabled; + + @Value("${actors.calculated_fields.debug_mode_rate_limits_per_tenant.configuration:50000:3600}") + private String calculatedFieldDebugPerTenantLimitsConfiguration; + @Autowired(required = false) private BuildProperties buildProperties; @@ -155,6 +161,9 @@ public class SystemInfoController extends BaseController { if (ruleChainDebugPerTenantLimitsEnabled) { systemParams.setRuleChainDebugPerTenantLimitsConfiguration(ruleChainDebugPerTenantLimitsConfiguration); } + if (calculatedFieldDebugPerTenantLimitsEnabled) { + systemParams.setCalculatedFieldDebugPerTenantLimitsConfiguration(calculatedFieldDebugPerTenantLimitsConfiguration); + } } systemParams.setMobileQrEnabled(Optional.ofNullable(qrCodeSettingService.findQrCodeSettings(TenantId.SYS_TENANT_ID)) .map(QrCodeSettings::getQrCodeConfig).map(QRCodeConfig::isShowOnHomePage) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java index cdad1145e8..1ecf38746e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java @@ -43,9 +43,9 @@ public class TsRollingArgumentEntry implements ArgumentEntry { private TreeMap tsRecords = new TreeMap<>(); public TsRollingArgumentEntry(List kvEntries, int limit, long timeWindow) { - kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); this.limit = limit; this.timeWindow = timeWindow; + kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry)); } public TsRollingArgumentEntry(TreeMap tsRecords, int limit, long timeWindow) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 30a3e51d8c..35e8c0e970 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -510,6 +510,12 @@ actors: js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" + calculated_fields: + debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all calculated fields per tenant + enabled: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more than 50 thousand events per hour + configuration: "${ACTORS_CALCULATED_FIELD_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" debug: settings: diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java index 30ad7a6507..0227365ed7 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldStateTest.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedField import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BasicKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.service.cf.CalculatedFieldResult; @@ -57,7 +57,7 @@ public class ScriptCalculatedFieldStateTest { private final DeviceId DEVICE_ID = new DeviceId(UUID.fromString("5512071d-5abc-411d-a907-4cdb6539c2eb")); private final AssetId ASSET_ID = new AssetId(UUID.fromString("5bc010ae-bcfd-46c8-98b9-8ee8c8955a76")); - private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new LongDataEntry("assetHumidity", 43L), 122L); + private final SingleValueArgumentEntry assetHumidityArgEntry = new SingleValueArgumentEntry(System.currentTimeMillis() - 10, new DoubleDataEntry("assetHumidity", 43.0), 122L); private final TsRollingArgumentEntry deviceTemperatureArgEntry = createRollingArgEntry(); private final long ts = System.currentTimeMillis(); @@ -127,52 +127,7 @@ public class ScriptCalculatedFieldStateTest { Output output = getCalculatedFieldConfig().getOutput(); assertThat(result.getType()).isEqualTo(output.getType()); assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 13.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenOldTelemetry() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - - TreeMap values = new TreeMap<>(); - values.put(ts - 40000, 4.0);// will not be used for calculation - values.put(ts - 45000, 2.0);// will not be used for calculation - values.put(ts - 20, 0.0); - - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); - } - - @Test - void testPerformCalculationWhenArgumentsMoreThanLimit() throws ExecutionException, InterruptedException { - TsRollingArgumentEntry argumentEntry = new TsRollingArgumentEntry(); - TreeMap values = new TreeMap<>(); - values.put(ts - 20, 1000.0);// will not be used - values.put(ts - 18, 0.0); - values.put(ts - 16, 0.0); - values.put(ts - 14, 0.0); - values.put(ts - 12, 0.0); - values.put(ts - 10, 0.0); - argumentEntry.setTsRecords(values); - - state.arguments = new HashMap<>(Map.of("deviceTemperature", argumentEntry, "assetHumidity", assetHumidityArgEntry)); - - CalculatedFieldResult result = state.performCalculation(ctx).get(); - - assertThat(result).isNotNull(); - Output output = getCalculatedFieldConfig().getOutput(); - assertThat(result.getType()).isEqualTo(output.getType()); - assertThat(result.getScope()).isEqualTo(output.getScope()); - assertThat(result.getResultMap()).isEqualTo(Map.of("averageDeviceTemperature", 0.0, "assetHumidity", 43L)); + assertThat(result.getResultMap()).isEqualTo(Map.of("maxDeviceTemperature", 17.0, "assetHumidity", 43.0)); } @Test @@ -189,7 +144,7 @@ public class ScriptCalculatedFieldStateTest { @Test void testIsReadyWhenEmptyEntryPresents() { -// state.arguments = new HashMap<>(Map.of("deviceTemperature", TsRollingArgumentEntry.EMPTY, "assetHumidity", assetHumidityArgEntry)); + state.arguments = new HashMap<>(Map.of("deviceTemperature", new TsRollingArgumentEntry(5, 30000L), "assetHumidity", assetHumidityArgEntry)); assertThat(state.isReady()).isFalse(); } @@ -235,7 +190,7 @@ public class ScriptCalculatedFieldStateTest { config.setArguments(Map.of("deviceTemperature", argument1, "assetHumidity", argument2)); - config.setExpression("var result = 0; foreach(element : deviceTemperature.entrySet()) { result += element.getValue(); } var map = {}; map.put(\"averageDeviceTemperature\", result / deviceTemperature.size()); map.put(\"assetHumidity\", assetHumidity); return map;"); + config.setExpression("return {\"maxDeviceTemperature\": deviceTemperature.max(), \"assetHumidity\": assetHumidity.value}"); Output output = new Output(); output.setType(OutputType.ATTRIBUTES); diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java index d87e690478..c451ffcbfc 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldStateTest.java @@ -117,10 +117,10 @@ public class SimpleCalculatedFieldStateTest { "key2", key2ArgEntry )); -// Map newArgs = Map.of("key3", TsRollingArgumentEntry.EMPTY); -// assertThatThrownBy(() -> state.updateState(newArgs)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Rolling argument entry is not supported for simple calculated fields."); + Map newArgs = Map.of("key3", new TsRollingArgumentEntry(10, 30000L)); + assertThatThrownBy(() -> state.updateState(newArgs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Rolling argument entry is not supported for simple calculated fields."); } @Test @@ -175,7 +175,7 @@ public class SimpleCalculatedFieldStateTest { "key1", key1ArgEntry, "key2", key2ArgEntry )); -// state.getArguments().put("key3", SingleValueArgumentEntry.EMPTY); + state.getArguments().put("key3", new SingleValueArgumentEntry()); assertThat(state.isReady()).isFalse(); } diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java index 57e4655d95..2514b9f34e 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntryTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.kv.LongDataEntry; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class SingleValueArgumentEntryTest { @@ -39,9 +40,9 @@ public class SingleValueArgumentEntryTest { @Test void testUpdateEntryWhenRollingEntryPassed() { -// assertThatThrownBy(() -> entry.updateEntry(TsRollingArgumentEntry.EMPTY)) -// .isInstanceOf(IllegalArgumentException.class) -// .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); + assertThatThrownBy(() -> entry.updateEntry(new TsRollingArgumentEntry(5, 30000L))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported argument entry type for single value argument entry: " + ArgumentEntryType.TS_ROLLING); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java index e4d5a59d03..75aca61825 100644 --- a/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntryTest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.cf.ctx.state; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.thingsboard.server.common.data.kv.BasicKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; @@ -40,7 +39,7 @@ public class TsRollingArgumentEntryTest { values.put(ts - 30, 12.0); values.put(ts - 20, 17.0); -// entry = new TsRollingArgumentEntry(values); + entry = new TsRollingArgumentEntry(5, 30000L, values); } @Test @@ -57,18 +56,10 @@ public class TsRollingArgumentEntryTest { assertThat(entry.getTsRecords().get(ts - 10)).isEqualTo(23.0); } - @Test - void testUpdateEntryWhenSingleValueEntryWithTheSameTsPassed() { - SingleValueArgumentEntry newEntry = new SingleValueArgumentEntry(ts - 20, new DoubleDataEntry("key", 23.0), 123L); - - assertThat(entry.updateEntry(newEntry)).isFalse(); - } - @Test void testUpdateEntryWhenRollingEntryPassed() { TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); TreeMap values = new TreeMap<>(); - values.put(ts - 20, 16.0); values.put(ts - 10, 7.0); values.put(ts - 5, 1.0); newEntry.setTsRecords(values); @@ -76,11 +67,11 @@ public class TsRollingArgumentEntryTest { assertThat(entry.updateEntry(newEntry)).isTrue(); assertThat(entry.getTsRecords()).hasSize(5); assertThat(entry.getTsRecords()).isEqualTo(Map.of( - ts - 40, new DoubleDataEntry("key", 10.0), - ts - 30, new DoubleDataEntry("key", 12.0), - ts - 20, new DoubleDataEntry("key", 17.0), - ts - 10, new DoubleDataEntry("key", 7.0), - ts - 5, new DoubleDataEntry("key", 1.0) + ts - 40, 10.0, + ts - 30, 12.0, + ts - 20, 17.0, + ts - 10, 7.0, + ts - 5, 1.0 )); } @@ -90,7 +81,44 @@ public class TsRollingArgumentEntryTest { assertThatThrownBy(() -> entry.updateEntry(newEntry)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Argument type " + ArgumentEntryType.TS_ROLLING + " only supports numeric values."); + .hasMessage("Time series rolling arguments supports only numeric values."); + } + + @Test + void testUpdateEntryWhenOldTelemetry() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 40000, 4.0);// will not be used for calculation + values.put(ts - 45000, 2.0);// will not be used for calculation + values.put(ts - 5, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(1); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 5, 0.0 + )); + } + + @Test + void testPerformCalculationWhenArgumentsMoreThanLimit() { + TsRollingArgumentEntry newEntry = new TsRollingArgumentEntry(); + TreeMap values = new TreeMap<>(); + values.put(ts - 20, 1000.0);// will not be used + values.put(ts - 18, 0.0); + values.put(ts - 16, 0.0); + values.put(ts - 14, 0.0); + newEntry.setTsRecords(values); + + entry = new TsRollingArgumentEntry(3, 30000L); + assertThat(entry.updateEntry(newEntry)).isTrue(); + assertThat(entry.getTsRecords()).hasSize(3); + assertThat(entry.getTsRecords()).isEqualTo(Map.of( + ts - 18, 0.0, + ts - 16, 0.0, + ts - 14, 0.0 + )); } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java index abe1932327..53f39977c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java @@ -34,4 +34,5 @@ public class SystemParams { boolean mobileQrEnabled; int maxDebugModeDurationMinutes; String ruleChainDebugPerTenantLimitsConfiguration; + String calculatedFieldDebugPerTenantLimitsConfiguration; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java index e0599db358..9b64dd0d40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/CalculatedFieldDebugEvent.java @@ -91,5 +91,4 @@ public class CalculatedFieldDebugEvent extends Event { return eventInfo; } - } From 4bd0833eed0df0c694e99abb39ded6d22cc8922e Mon Sep 17 00:00:00 2001 From: mpetrov Date: Fri, 14 Feb 2025 18:39:45 +0200 Subject: [PATCH 213/438] Added Calculated field arguments autocomplete --- .../calculated-fields-table-config.ts | 29 +++++-- .../calculated-field-dialog.component.html | 1 + .../calculated-field-dialog.component.ts | 6 ++ ...ed-field-script-test-dialog.component.html | 1 + .../shared/models/calculated-field.models.ts | 79 +++++++++++++++++++ 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index e7ad697262..8f967b68ce 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -36,9 +36,14 @@ import { EntityDebugSettingsPanelComponent } from '@home/components/entity/debug import { CalculatedFieldsService } from '@core/http/calculated-fields.service'; import { catchError, filter, switchMap, tap } from 'rxjs/operators'; import { + ArgumentType, CalculatedField, + CalculatedFieldArgument, + CalculatedFieldEventArguments, CalculatedFieldDebugDialogData, CalculatedFieldDialogData, + CalculatedFieldRollingValueArgumentAutocomplete, + CalculatedFieldSingleValueArgumentAutocomplete, CalculatedFieldTestScriptDialogData, } from '@shared/models/calculated-field.models'; import { @@ -48,6 +53,7 @@ import { } from './components/public-api'; import { ImportExportService } from '@shared/import-export/import-export.service'; import { isObject } from '@core/utils'; +import { TbEditorCompleter } from '@shared/models/ace/completion.models'; export class CalculatedFieldsTableConfig extends EntityTableConfig { @@ -58,7 +64,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog.call(this, calculatedField), + action: (calculatedField: CalculatedField) => this.openDebugEventsDialog.call(this, calculatedField), }; constructor(private calculatedFieldsService: CalculatedFieldsService, @@ -122,7 +128,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig true, - onAction: (_, entity) => this.openDebugDialog(entity), + onAction: (_, entity) => this.openDebugEventsDialog(entity), }, { name: '', @@ -149,7 +155,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.openDebugDialog(calculatedField) + action: () => this.openDebugEventsDialog(calculatedField) }; const { viewContainerRef } = this.getTable(); if ($event) { @@ -211,14 +217,15 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig(CalculatedFieldDebugDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -267,7 +274,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.updateData()); } - private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: Record, openCalculatedFieldEdit = true): Observable { + private getTestScriptDialog(calculatedField: CalculatedField, argumentsObj?: CalculatedFieldEventArguments, openCalculatedFieldEdit = true): Observable { const resultArguments = Object.keys(calculatedField.configuration.arguments).reduce((acc, key) => { acc[key] = isObject(argumentsObj) && argumentsObj.hasOwnProperty(key) ? argumentsObj[key] : ''; return acc; @@ -279,6 +286,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig): TbEditorCompleter { + return new TbEditorCompleter(Object.keys(argumentsObj).reduce((acc, key) => { + acc[key] = argumentsObj[key].refEntityKey.type === ArgumentType.Rolling + ? CalculatedFieldRollingValueArgumentAutocomplete + : CalculatedFieldSingleValueArgumentAutocomplete; + return acc; + }, {})) + } } diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 2f707c0a2f..f1e6905aab 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -99,6 +99,7 @@ [functionArgs]="functionArgs$ | async" [disableUndefinedCheck]="true" [scriptLanguage]="ScriptLanguage.TBEL" + [editorCompleter]="argumentsEditorCompleter$ | async" helpId="calculated-field/expression_fn" >