From e5311984bba0d6449e6db65c9fdfac22cf2e98b7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 7 Feb 2024 15:24:40 +0200 Subject: [PATCH 01/72] Housekeeper API --- .../DefaultHousekeeperService.java | 144 ++++++++++++++++++ .../HousekeeperReprocessingService.java | 134 ++++++++++++++++ .../AttributesDeletionTaskProcessor.java | 41 +++++ .../EntityDeletionTaskProcessor.java | 37 +++++ .../EventsDeletionTaskProcessor.java | 38 +++++ .../processor/HousekeeperTaskProcessor.java | 27 ++++ .../TelemetryDeletionTaskProcessor.java | 41 +++++ .../src/main/resources/thingsboard.yml | 7 + common/proto/src/main/proto/queue.proto | 8 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 + .../provider/AwsSqsMonolithQueueFactory.java | 21 +++ .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++ .../InMemoryMonolithQueueFactory.java | 20 +++ .../provider/KafkaMonolithQueueFactory.java | 51 ++++++- .../provider/KafkaTbCoreQueueFactory.java | 50 ++++++ .../provider/PubSubMonolithQueueFactory.java | 21 +++ .../provider/PubSubTbCoreQueueFactory.java | 21 +++ .../RabbitMqMonolithQueueFactory.java | 21 +++ .../provider/RabbitMqTbCoreQueueFactory.java | 20 +++ .../ServiceBusMonolithQueueFactory.java | 21 +++ .../ServiceBusTbCoreQueueFactory.java | 21 +++ .../queue/provider/TbCoreQueueFactory.java | 11 ++ .../provider/TbCoreQueueProducerProvider.java | 17 +++ .../provider/TbQueueProducerProvider.java | 6 + .../TbRuleEngineProducerProvider.java | 11 ++ .../TbTransportQueueProducerProvider.java | 10 ++ .../TbVersionControlProducerProvider.java | 10 ++ .../queue/settings/TbQueueCoreSettings.java | 6 + .../server/dao/device/DeviceServiceImpl.java | 5 + .../dao/entity/AbstractEntityService.java | 6 + .../dao/housekeeper/HousekeeperService.java | 24 +++ .../dao/housekeeper/data/HousekeeperTask.java | 51 +++++++ .../housekeeper/data/HousekeeperTaskType.java | 23 +++ 34 files changed, 951 insertions(+), 3 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java new file mode 100644 index 0000000000..7c7e72b511 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.ByteString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultHousekeeperService implements HousekeeperService { + + private final Map taskProcessors; + + private final TbQueueConsumer> consumer; + private final TbQueueProducer> producer; + private final HousekeeperReprocessingService reprocessingService; + private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private boolean stopped; + + public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider, + DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.reprocessingService = reprocessingService; + this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); + this.dataDecodingEncodingService = dataDecodingEncodingService; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + + for (TbProtoQueueMsg msg : msgs) { + try { + processTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper service"); + } + + protected void processTask(TbProtoQueueMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + log.error("Unsupported task type {}: {}", task.getTaskType(), task); + return; + } + + log.info("[{}] Processing task: {}", task.getTenantId(), task); + try { + taskProcessor.process(task); + } catch (Exception e) { + log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + reprocessingService.submitForReprocessing(msg); + } + } + + @Override + public void submitTask(HousekeeperTask task) { + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + .setTask(HousekeeperTaskProto.newBuilder() + .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .build()) + .build()), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper service"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java new file mode 100644 index 0000000000..234cd07c36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgHeaders; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; +import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperReprocessingService { + + private final DefaultHousekeeperService housekeeperService; + private final TbQueueProducerProvider producerProvider; + private final TbQueueConsumer> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + + @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + private int pollInterval; + + private final long startTs = System.currentTimeMillis(); + private boolean stopped; + // todo: stats + + public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, + TbCoreQueueFactory queueFactory, + TbQueueProducerProvider producerProvider) { + this.housekeeperService = housekeeperService; + this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.producerProvider = producerProvider; + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumerExecutor.submit(() -> { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + stop(); + return; + } + + for (TbProtoQueueMsg msg : msgs) { + long msgTs = Uuids.unixTimestamp(msg.getKey()); + if (msgTs >= startTs) { + stop(); + return; // fixme: we should commit already reprocessed messages + } + + try { + reprocessTask(msg); + } catch (Exception e) { + log.error("Message processing failed", e); + } + } + consumer.commit(); + } catch (Throwable t) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", t); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + }); + log.info("Started Housekeeper tasks reprocessing"); + } + + private void reprocessTask(TbProtoQueueMsg msg) { + housekeeperService.processTask(msg);// fixme: or should we submit to queue? + } + + public void submitForReprocessing(TbProtoQueueMsg msg) { + TbQueueMsgHeaders msgHeaders = msg.getHeaders(); + long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) + .map(header -> bytesToLong(header)) + .orElse(0L); + reprocessingAttempts++; + msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + + var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + } + + @PreDestroy + private void stop() { + log.info("Stopped Housekeeper tasks reprocessing"); + stopped = true; + consumer.unsubscribe(); + consumerExecutor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java new file mode 100644 index 0000000000..34bd95a901 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AttributesService attributesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java new file mode 100644 index 0000000000..e1b0877d2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { + + @Override + public void process(HousekeeperTask task) throws Exception { + + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java new file mode 100644 index 0000000000..7b8fdb1242 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; + + @Override + public void process(HousekeeperTask task) throws Exception { + eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_EVENTS; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java new file mode 100644 index 0000000000..bfa7466dc3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +public interface HousekeeperTaskProcessor { + + void process(HousekeeperTask task) throws Exception; + + HousekeeperTaskType getTaskType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java new file mode 100644 index 0000000000..feb5b0ddc1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +@Component +@RequiredArgsConstructor +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(HousekeeperTask task) throws Exception { +// timeseriesService.remove() +// timeseriesService.removeAllLatest(); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 70a5146bb7..17b4c85cfb 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1383,6 +1383,9 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1404,6 +1407,7 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1559,6 +1563,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + topic: "tb_housekeeper" + poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 4da2883291..0dbd96a53d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1391,3 +1391,11 @@ message LifecycleEventProto { bool success = 7; string error = 8; } + +message ToHousekeeperServiceMsg { + HousekeeperTaskProto task = 1; +} + +message HousekeeperTaskProto { + bytes value = 1; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index f11eaaef48..9340cafa67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - protected static byte[] longToBytes(long x) { + public static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - protected static long bytesToLong(byte[] bytes) { + public static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d23d92ee26..d3a5eb821d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -43,6 +43,8 @@ public class TbKafkaTopicConfigs { private String fwUpdatesProperties; @Value("${queue.kafka.topic-properties.version-control:}") private String vcProperties; + @Value("${queue.kafka.topic-properties.housekeeper:}") + private String housekeeperProperties; @Getter private Map coreConfigs; @@ -62,6 +64,8 @@ public class TbKafkaTopicConfigs { private Map fwUpdatesConfigs; @Getter private Map vcConfigs; + @Getter + private Map housekeeperConfigs; @PostConstruct private void init() { @@ -76,6 +80,7 @@ public class TbKafkaTopicConfigs { jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); + housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 8a127384be..5a61bc1d80 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -226,6 +227,26 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 6c69b2ec10..a1be25f29d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -205,6 +206,26 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a681038aa9..1ee8b404a5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -160,6 +160,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 662d29af5b..5ee9b6f53b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +40,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -82,6 +83,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -115,6 +117,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -346,6 +349,52 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index db6ee52181..c01ba32f70 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -80,6 +81,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -112,6 +114,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -303,6 +306,53 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 62fe40f853..a63a9fc16b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -227,6 +228,26 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 0e7e1ba241..5f1f282e39 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 1b7f705175..802b08773d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -224,6 +225,26 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e0d0ea3b3a..ceb6e03ca5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -198,6 +198,26 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 359cbf67b8..1e9c1cadaf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -223,6 +224,26 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index d2ef072ffa..e39df45b82 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -198,6 +199,26 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return null; + } + + @Override + public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + return null; + } + + @Override + public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 5aa7866033..279a469415 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -130,4 +132,13 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * @return */ TbQueueProducer> createVersionControlMsgProducer(); + + TbQueueProducer> createHousekeeperMsgProducer(); + + TbQueueConsumer> createHousekeeperMsgConsumer(); + + TbQueueProducer> createHousekeeperDelayedMsgProducer(); + + TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 5a99f361b4..4fa22cb7e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -41,6 +43,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; + private TbQueueProducer> toHousekeeper; + private TbQueueProducer> toHousekeeperDelayed; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -55,6 +59,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); + this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); } @Override @@ -91,4 +97,15 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { return toVersionControl; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + return toHousekeeperDelayed; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 4dbdca4375..5cdf390f47 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -78,4 +79,9 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbVersionControlMsgProducer(); + + TbQueueProducer> getHousekeeperMsgProducer(); + + TbQueueProducer> getHousekeeperDelayedMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index e4d98796f4..c5ddf57fc0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -89,4 +89,15 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return null; // fixme + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index c83b905a7c..e8dd7431a6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -85,4 +85,14 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index dbbaa9f072..7ddd2109ba 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -81,4 +81,14 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0a8c12eb8d..4edb688235 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -34,6 +34,12 @@ public class TbQueueCoreSettings { @Value("${queue.core.usage-stats-topic:tb_usage_stats}") private String usageStatsTopic; + @Value("${queue.core.housekeeper.topic:tb_housekeeper}") + private String housekeeperTopic; + + @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + private String housekeeperDelayedTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f190923167..f0e5dc1a4a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -75,6 +75,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -328,6 +329,10 @@ public class DeviceServiceImpl extends AbstractCachedEntityService Date: Wed, 7 Feb 2024 16:12:11 +0200 Subject: [PATCH 02/72] Submit cleanup tasks on DeleteEntityEvent --- .../EntityAlarmsDeletionTaskProcessor.java | 40 +++++++++++++++++++ .../server/dao/alarm/AlarmService.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 1 - .../server/dao/asset/BaseAssetService.java | 3 -- .../dao/customer/CustomerServiceImpl.java | 1 - .../dao/dashboard/DashboardServiceImpl.java | 1 - .../dao/device/DeviceProfileServiceImpl.java | 1 - .../server/dao/device/DeviceServiceImpl.java | 8 ---- .../server/dao/edge/EdgeServiceImpl.java | 7 +--- .../dao/entity/AbstractEntityService.java | 31 +++++++++++--- .../dao/entityview/EntityViewServiceImpl.java | 1 - .../dao/housekeeper/data/HousekeeperTask.java | 9 +++++ .../housekeeper/data/HousekeeperTaskType.java | 4 +- .../server/dao/rule/BaseRuleChainService.java | 7 ++-- .../dao/tenant/TenantProfileServiceImpl.java | 1 - .../server/dao/tenant/TenantServiceImpl.java | 1 - .../server/dao/user/UserServiceImpl.java | 1 - 18 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java new file mode 100644 index 0000000000..f9e61fc1cb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { + + private final AlarmService alarmService; + + @Override + public void process(HousekeeperTask task) throws Exception { + alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITY_ALARMS; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 203fe8e744..7a55348bed 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -106,7 +106,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId); + void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 72e1b9c7bf..a14c172a57 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -189,7 +189,6 @@ public class BaseAlarmService extends AbstractCachedEntityService event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); + + cleanUpRelatedData(tenantId, entityId); + if (EntityType.USER.equals(entityId.getEntityType())) { +// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); +// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + } + } + + protected void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + // todo: skipped entities list + relationService.deleteEntityRelations(tenantId, entityId); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + } + protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); relationService.saveRelation(tenantId, relation); @@ -79,11 +103,6 @@ public abstract class AbstractEntityService { relationService.deleteRelation(tenantId, relation); } - protected void deleteEntityRelations(TenantId tenantId, EntityId entityId) { - relationService.deleteEntityRelations(tenantId, entityId); - alarmService.deleteEntityAlarmRelations(tenantId, entityId); - } - protected static Optional extractConstraintViolationException(Exception t) { if (t instanceof ConstraintViolationException) { return Optional.of((ConstraintViolationException) t); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index ed962ce84c..6940b42794 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -331,7 +331,6 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - deleteEntityRelations(tenantId, existingNode.getId()); + cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -771,7 +771,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); for (var node : ruleNodes) { - deleteEntityRelations(tenantId, node.getId()); + cleanUpRelatedData(tenantId, node.getId()); } ruleNodeDao.deleteByIdIn(ruleNodeIds); } @@ -783,7 +783,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); } - deleteEntityRelations(tenantId, ruleChainId); } @@ -820,8 +819,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } private void deleteRuleNode(TenantId tenantId, EntityId entityId) { - deleteEntityRelations(tenantId, entityId); ruleNodeDao.removeById(tenantId, entityId.getId()); + cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 31a419a5d0..2462ca19d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -130,7 +130,6 @@ public class TenantProfileServiceImpl extends AbstractCachedEntityService Date: Wed, 7 Feb 2024 16:50:03 +0200 Subject: [PATCH 03/72] Alarms unassign housekeeper task; remove in-memory housekeeper service --- .../entitiy/alarm/DefaultTbAlarmService.java | 16 +-- .../service/entitiy/alarm/TbAlarmService.java | 2 +- .../InMemoryHouseKeeperServiceService.java | 103 ------------------ .../AlarmsUnassignTaskProcessor.java | 41 +++++++ .../AttributesDeletionTaskProcessor.java | 3 +- .../EntityAlarmsDeletionTaskProcessor.java | 2 +- .../EntityDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 2 +- .../processor/HousekeeperTaskProcessor.java | 4 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../alarm/DefaultTbAlarmServiceTest.java | 2 +- .../dao/entity/AbstractEntityService.java | 6 +- .../data/AlarmsUnassignHousekeeperTask.java | 18 +-- .../dao/housekeeper/data/HousekeeperTask.java | 19 ++-- 14 files changed, 82 insertions(+), 140 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java rename common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java => dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java (59%) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 5305018b84..deefdaa614 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -176,16 +176,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { + public List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { List totalAlarmIds = new ArrayList<>(); PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); while (true) { - PageData pageData = alarmService.findAlarmIdsByAssigneeId(user.getTenantId(), user.getId(), pageLink); + PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); List alarmIds = pageData.getData(); if (alarmIds.isEmpty()) { break; } - processAlarmsUnassignment(tenantId, user, alarmIds, unassignTs); + processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs); totalAlarmIds.addAll(alarmIds); pageLink = pageLink.nextPageLink(); } @@ -204,16 +204,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb return ts > 0 ? ts : System.currentTimeMillis(); } - private void processAlarmsUnassignment(TenantId tenantId, User user, List alarmIds, long unassignTs) { + private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List alarmIds, long unassignTs) { for (AlarmId alarmId : alarmIds) { - log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, user.getId()); - AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(user.getTenantId(), alarmId, unassignTs); + log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId); + AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs); if (!result.isSuccessful()) { - log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, user.getId()); + log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId); continue; } if (result.isModified()) { - String comment = String.format("Alarm was unassigned because user %s - was deleted", user.getTitle()); + String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); notificationEntityService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 34dc3d11f2..c975291b58 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -41,7 +41,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java deleted file mode 100644 index 4870990606..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.housekeeper; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.HouseKeeperService; -import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@RequiredArgsConstructor -@Slf4j -public class InMemoryHouseKeeperServiceService implements HouseKeeperService { - - final TbAlarmService alarmService; - - ListeningExecutorService executor; - - AtomicInteger queueSize = new AtomicInteger(); - AtomicInteger totalProcessedCounter = new AtomicInteger(); - - @PostConstruct - public void init() { - log.debug("Starting HouseKeeper service"); - executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper"))); - } - - @PreDestroy - public void destroy() { - if (executor != null) { - log.debug("Stopping HouseKeeper service"); - executor.shutdown(); - } - } - - @TransactionalEventListener(fallbackExecution = true) - public void handleEvent(DeleteEntityEvent event) { - log.trace("[{}] DeleteEntityEvent handler: {}", event.getTenantId(), event); - EntityId entityId = event.getEntityId(); - if (EntityType.USER.equals(entityId.getEntityType())) { - unassignDeletedUserAlarms(event.getTenantId(), (User) event.getEntity(), event.getTs()); - } - } - - @Override - public ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { - log.debug("[{}][{}] unassignDeletedUserAlarms submitting, pending queue size: {} ", tenantId, user.getId().getId(), queueSize.get()); - queueSize.incrementAndGet(); - ListenableFuture> future = executor.submit(() -> alarmService.unassignDeletedUserAlarms(tenantId, user, unassignTs)); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(List alarmIds) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.debug("[{}][{}] unassignDeletedUserAlarms finished, pending queue size: {}, total processed count: {} ", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get()); - } - - @Override - public void onFailure(Throwable throwable) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.error("[{}][{}] unassignDeletedUserAlarms failed, pending queue size: {}, total processed count: {}", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get(), throwable); - } - }, MoreExecutors.directExecutor()); - return future; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java new file mode 100644 index 0000000000..54bf7989af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; + +@Component +@RequiredArgsConstructor +public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { + + private final TbAlarmService alarmService; + + @Override + public void process(AlarmsUnassignHousekeeperTask task) throws Exception { + alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.UNASSIGN_ALARMS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 34bd95a901..169ac3e573 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -17,14 +17,13 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AttributesService attributesService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java index f9e61fc1cb..a91f2262f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AlarmService alarmService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java index e1b0877d2c..deb693c466 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java @@ -22,7 +22,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { @Override public void process(HousekeeperTask task) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 7b8fdb1242..56edf01637 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final EventService eventService; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index bfa7466dc3..7daa99540e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -18,9 +18,9 @@ package org.thingsboard.server.service.housekeeper.processor; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; -public interface HousekeeperTaskProcessor { +public interface HousekeeperTaskProcessor { - void process(HousekeeperTask task) throws Exception; + void process(T task) throws Exception; HousekeeperTaskType getTaskType(); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index feb5b0ddc1..2a491148ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; @Component @RequiredArgsConstructor -public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 5d493193db..38b64d50ed 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -173,7 +173,7 @@ public class DefaultTbAlarmServiceTest { User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user, System.currentTimeMillis()); + service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index ba37737b84..7bfe76e753 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -24,6 +24,7 @@ import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -78,9 +79,8 @@ public abstract class AbstractEntityService { log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); - if (EntityType.USER.equals(entityId.getEntityType())) { -// housekeeperService.submitTask(HousekeeperTask.unassignAlarms(tenantId, entityId)); -// unassignDeletedUserAlarms(tenantId, (User) event.getEntity(), event.getTs()); + if (entityId.getEntityType() == EntityType.USER) { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java similarity index 59% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java rename to dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index 55b164df52..876a8f736b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -13,17 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper; +package org.thingsboard.server.dao.housekeeper.data; -import com.google.common.util.concurrent.ListenableFuture; +import lombok.Getter; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.TenantId; -import java.util.List; +@Getter +public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { -public interface HouseKeeperService { + private final String userTitle; + private final long ts; - ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + protected AlarmsUnassignHousekeeperTask(User user) { + super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); + this.userTitle = user.getTitle(); + this.ts = System.currentTimeMillis(); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 7ab415fe7b..726e07a790 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,27 +15,28 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Data; +import lombok.Getter; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UserId; import java.io.Serializable; /* * on start, read the retry queue and put the messages back to main queue (save offset) * */ -@Data +@Getter public class HousekeeperTask implements Serializable { private final TenantId tenantId; private final EntityId entityId; private final HousekeeperTaskType taskType; - // maybe we should not delete relations asynchronously -// public static HousekeeperTask deleteRelations(TenantId tenantId, EntityId entityId) { -// return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_RELATIONS); -// } + protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { + this.tenantId = tenantId; + this.entityId = entityId; + this.taskType = taskType; + } public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); @@ -49,8 +50,8 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } - public static HousekeeperTask unassignAlarms(TenantId tenantId, UserId userId) { - return new HousekeeperTask(tenantId, userId, HousekeeperTaskType.UNASSIGN_ALARMS); + public static HousekeeperTask unassignAlarms(User user) { + return new AlarmsUnassignHousekeeperTask(user); } public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId entityId) { From 7e07771accb561d487a57f28f6e72b8e2a8437b1 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 14 Feb 2024 20:20:08 +0200 Subject: [PATCH 04/72] Housekeeper tasks reprocessing --- .../DefaultHousekeeperService.java | 42 ++++++++----- .../HousekeeperReprocessingService.java | 50 ++++++++-------- .../AlarmsUnassignTaskProcessor.java | 8 ++- .../TelemetryDeletionTaskProcessor.java | 2 +- .../src/main/resources/thingsboard.yml | 1 + common/proto/src/main/proto/queue.proto | 2 + .../queue/common/AbstractTbQueueTemplate.java | 4 +- .../queue/settings/TbQueueCoreSettings.java | 2 +- .../dao/entity/AbstractEntityService.java | 31 +--------- .../dao/housekeeper/CleanUpService.java | 60 +++++++++++++++++++ .../data/AlarmsUnassignHousekeeperTask.java | 2 - .../dao/housekeeper/data/HousekeeperTask.java | 9 ++- .../server/dao/rule/BaseRuleChainService.java | 6 +- .../dao/sql/attributes/JpaAttributeDao.java | 21 +++++-- 14 files changed, 153 insertions(+), 87 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 7c7e72b511..f669df5305 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.housekeeper; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -40,6 +40,8 @@ import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProce import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -49,7 +51,7 @@ import java.util.stream.Collectors; @Slf4j public class DefaultHousekeeperService implements HousekeeperService { - private final Map taskProcessors; + private final Map> taskProcessors; private final TbQueueConsumer> consumer; private final TbQueueProducer> producer; @@ -65,7 +67,8 @@ public class DefaultHousekeeperService implements HousekeeperService { public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, - DataDecodingEncodingService dataDecodingEncodingService, List taskProcessors) { + DataDecodingEncodingService dataDecodingEncodingService, + @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; @@ -86,7 +89,7 @@ public class DefaultHousekeeperService implements HousekeeperService { for (TbProtoQueueMsg msg : msgs) { try { - processTask(msg); + processTask(msg.getValue()); } catch (Exception e) { log.error("Message processing failed", e); } @@ -107,19 +110,18 @@ public class DefaultHousekeeperService implements HousekeeperService { log.info("Started Housekeeper service"); } - protected void processTask(TbProtoQueueMsg msg) { - HousekeeperTask task = dataDecodingEncodingService.decode(msg.getValue().getTask().getValue().toByteArray()).get(); - HousekeeperTaskProcessor taskProcessor = taskProcessors.get(task.getTaskType()); - if (taskProcessor == null) { - log.error("Unsupported task type {}: {}", task.getTaskType(), task); - return; - } + @SuppressWarnings("unchecked") + protected void processTask(ToHousekeeperServiceMsg msg) { + HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); + HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); - log.info("[{}] Processing task: {}", task.getTenantId(), task); + log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); try { - taskProcessor.process(task); + taskProcessor.process((T) task); } catch (Exception e) { - log.error("[{}] Task processing failed: {}", task.getTenantId(), task, e); + log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", + task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + task.getTaskType(), msg.getTask().getAttempt(), task, e); reprocessingService.submitForReprocessing(msg); } } @@ -127,11 +129,20 @@ public class DefaultHousekeeperService implements HousekeeperService { @Override public void submitTask(HousekeeperTask task) { TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), ToHousekeeperServiceMsg.newBuilder() + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .setTs(task.getTs()) + .setAttempt(0) .build()) .build()), null); + log.trace("[{}][{}][{}] Submitted task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + } + + @SuppressWarnings("unchecked") + private HousekeeperTaskProcessor getTaskProcessor(HousekeeperTaskType taskType) { + return Optional.ofNullable((HousekeeperTaskProcessor) taskProcessors.get(taskType)) + .orElseThrow(() -> new IllegalArgumentException("Unsupported task type " + taskType)); } @PreDestroy @@ -141,4 +152,5 @@ public class DefaultHousekeeperService implements HousekeeperService { consumer.unsubscribe(); consumerExecutor.shutdownNow(); } + } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 234cd07c36..4fca14e44f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -15,16 +15,15 @@ */ package org.thingsboard.server.service.housekeeper; -import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueMsgHeaders; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -33,13 +32,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PreDestroy; import java.util.List; -import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.bytesToLong; -import static org.thingsboard.server.queue.common.AbstractTbQueueTemplate.longToBytes; - @TbCoreComponent @Service @Slf4j @@ -77,15 +73,21 @@ public class HousekeeperReprocessingService { return; } - for (TbProtoQueueMsg msg : msgs) { - long msgTs = Uuids.unixTimestamp(msg.getKey()); - if (msgTs >= startTs) { - stop(); - return; // fixme: we should commit already reprocessed messages + if (msgs.stream().anyMatch(msg -> { + boolean newMsg = msg.getValue().getTask().getTs() >= startTs; + if (newMsg) { + log.info("Stopping reprocessing due to msg is new {}", msg); } - + return newMsg; + })) { + stop(); // fixme: we should commit already reprocessed messages; maybe submit for reprocessing again and commit? + // msg batch size should be 1. otherwise some tasks won't be reprocessed + return; + } + for (TbProtoQueueMsg msg : msgs) { try { - reprocessTask(msg); + housekeeperService.processTask(msg.getValue());// fixme: or should we submit to queue? + Thread.sleep(1000); } catch (Exception e) { log.error("Message processing failed", e); } @@ -106,21 +108,19 @@ public class HousekeeperReprocessingService { log.info("Started Housekeeper tasks reprocessing"); } - private void reprocessTask(TbProtoQueueMsg msg) { - housekeeperService.processTask(msg);// fixme: or should we submit to queue? - } - - public void submitForReprocessing(TbProtoQueueMsg msg) { - TbQueueMsgHeaders msgHeaders = msg.getHeaders(); - long reprocessingAttempts = Optional.ofNullable(msgHeaders.get("reprocessingAttempts")) - .map(header -> bytesToLong(header)) - .orElse(0L); - reprocessingAttempts++; - msgHeaders.put("reprocessingAttempts", longToBytes(reprocessingAttempts)); + public void submitForReprocessing(ToHousekeeperServiceMsg msg) { + HousekeeperTaskProto task = msg.getTask(); + int attempt = task.getAttempt() + 1; + msg = msg.toBuilder() + .setTask(task.toBuilder() + .setAttempt(attempt) + .setTs(System.currentTimeMillis()) + .build()) + .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(Uuids.timeBased(), msg.getValue(), msgHeaders), null); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 54bf7989af..247466af65 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -16,21 +16,27 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; +import java.util.List; + @Component @RequiredArgsConstructor +@Slf4j public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { private final TbAlarmService alarmService; @Override public void process(AlarmsUnassignHousekeeperTask task) throws Exception { - alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + List alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index 2a491148ea..e925c9f20c 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -35,7 +35,7 @@ public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor< @Override public HousekeeperTaskType getTaskType() { - return HousekeeperTaskType.DELETE_ATTRIBUTES; + return HousekeeperTaskType.DELETE_TELEMETRY; } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 17b4c85cfb..6a96c0a3db 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1565,6 +1565,7 @@ queue: print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: topic: "tb_housekeeper" + reprocessing-topic: "tb_housekeeper.reprocessing" poll-interval-ms: "1000" vc: # Default topic name for Kafka, RabbitMQ, etc. diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 0dbd96a53d..c4e8e64e42 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1398,4 +1398,6 @@ message ToHousekeeperServiceMsg { message HousekeeperTaskProto { bytes value = 1; + int64 ts = 2; + int32 attempt = 3; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java index 9340cafa67..f11eaaef48 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueTemplate.java @@ -46,13 +46,13 @@ public class AbstractTbQueueTemplate { return new String(data, StandardCharsets.UTF_8); } - public static byte[] longToBytes(long x) { + protected static byte[] longToBytes(long x) { ByteBuffer longBuffer = ByteBuffer.allocate(Long.BYTES); longBuffer.putLong(0, x); return longBuffer.array(); } - public static long bytesToLong(byte[] bytes) { + protected static long bytesToLong(byte[] bytes) { return ByteBuffer.wrap(bytes).getLong(); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 4edb688235..0b66c2fa75 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -37,7 +37,7 @@ public class TbQueueCoreSettings { @Value("${queue.core.housekeeper.topic:tb_housekeeper}") private String housekeeperTopic; - @Value("${queue.core.housekeeper.topic:tb_housekeeper.delayed}") + @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}") private String housekeeperDelayedTopic; @Value("${queue.core.partitions}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 7bfe76e753..6b88f19699 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -20,11 +20,8 @@ import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -33,10 +30,8 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.housekeeper.HousekeeperService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.CleanUpService; import org.thingsboard.server.dao.relation.RelationService; import java.util.Collections; @@ -70,28 +65,8 @@ public abstract class AbstractEntityService { protected EdgeService edgeService; @Autowired - protected HousekeeperService housekeeperService; - - @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService - public void onEntityDeleted(DeleteEntityEvent event) { - TenantId tenantId = event.getTenantId(); - EntityId entityId = event.getEntityId(); - log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); - - cleanUpRelatedData(tenantId, entityId); - if (entityId.getEntityType() == EntityType.USER) { - housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); - } - } - - protected void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { - // todo: skipped entities list - relationService.deleteEntityRelations(tenantId, entityId); - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); - } + @Lazy + protected CleanUpService cleanUpService; protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java new file mode 100644 index 0000000000..0086e17926 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.relation.RelationService; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CleanUpService { + + private final HousekeeperService housekeeperService; + private final RelationService relationService; + + @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService + public void handleEntityDeletionEvent(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); + + log.info("[{}][{}][{}] Handling DeleteEntityEvent", tenantId, entityId.getEntityType(), entityId.getId()); + cleanUpRelatedData(tenantId, entityId); + if (entityId.getEntityType() == EntityType.USER) { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + } + } + + public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + // todo: skipped entities list + relationService.deleteEntityRelations(tenantId, entityId); + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index 876a8f736b..aa1858f59f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -22,12 +22,10 @@ import org.thingsboard.server.common.data.User; public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { private final String userTitle; - private final long ts; protected AlarmsUnassignHousekeeperTask(User user) { super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); this.userTitle = user.getTitle(); - this.ts = System.currentTimeMillis(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 726e07a790..9fcbbfe43d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,27 +15,26 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import java.io.Serializable; -/* - * on start, read the retry queue and put the messages back to main queue (save offset) - * */ -@Getter +@Data public class HousekeeperTask implements Serializable { private final TenantId tenantId; private final EntityId entityId; private final HousekeeperTaskType taskType; + private final long ts; protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { this.tenantId = tenantId; this.entityId = entityId; this.taskType = taskType; + this.ts = System.currentTimeMillis(); } public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 88fa6f1351..ebbec8969f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -177,7 +177,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? + cleanUpService.cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -771,7 +771,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); for (var node : ruleNodes) { - cleanUpRelatedData(tenantId, node.getId()); + cleanUpService.cleanUpRelatedData(tenantId, node.getId()); } ruleNodeDao.deleteByIdIn(ruleNodeIds); } @@ -820,7 +820,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNode(TenantId tenantId, EntityId entityId) { ruleNodeDao.removeById(tenantId, entityId.getId()); - cleanUpRelatedData(tenantId, entityId); + cleanUpService.cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index cb51a7d8ad..d8cf6aab32 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -20,9 +20,11 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -31,6 +33,7 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; @@ -132,10 +135,10 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Override public List findAll(TenantId tenantId, EntityId entityId, String attributeType) { return DaoUtil.convertDataList(Lists.newArrayList( - attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( - entityId.getEntityType(), - entityId.getId(), - attributeType))); + attributeKvRepository.findAllByEntityTypeAndEntityIdAndAttributeType( + entityId.getEntityType(), + entityId.getId(), + attributeType))); } @Override @@ -188,6 +191,16 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return futuresList; } + @Transactional + @Override + public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_type = ? and entity_id = ? " + + "RETURNING attribute_type, attribute_key", entityId.getEntityType().name(), entityId.getId()).stream() + .map(deleted -> Pair.of((String) deleted.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), + (String) deleted.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) + .collect(Collectors.toList()); + } + private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, String attributeType, String attributeKey) { return new AttributeKvCompositeKey( entityId.getEntityType(), From a1bede3cbb4ab914b21b05c37b51afd0006809a8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 14 Feb 2024 20:20:26 +0200 Subject: [PATCH 05/72] Attributes deletion task processor --- .../AttributesDeletionTaskProcessor.java | 5 +- .../dao/attributes/AttributesService.java | 2 + .../server/dao/attributes/AttributesDao.java | 4 + .../dao/attributes/BaseAttributesService.java | 8 ++ .../attributes/CachedAttributesService.java | 94 +++++++++++-------- 5 files changed, 71 insertions(+), 42 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 169ac3e573..e3bac35e9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; @@ -23,13 +24,15 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor +@Slf4j public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AttributesService attributesService; @Override public void process(HousekeeperTask task) throws Exception { -// attributesService.removeAll(task.getTenantId(), task.getEntityId(), DataConstants.CLIENT_SCOPE); + int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); + log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 3dd9626739..4f0840c704 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -49,4 +49,6 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds, String scope); + int removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 08a9419885..8cfc6be57b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.commons.lang3.tuple.Pair; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -46,4 +47,7 @@ public interface AttributesDao { List findAllKeysByEntityIds(TenantId tenantId, EntityType entityType, List entityIds); List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, EntityType entityType, List entityIds, String attributeType); + + List> removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 342cb26485..1cace5b71c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -113,4 +114,11 @@ public class BaseAttributesService implements AttributesService { validate(entityId, scope); return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } + + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + return deleted.size(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 1666015a48..1751d33312 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -147,47 +148,47 @@ public class CachedAttributesService implements AttributesService { return Futures.transformAsync(cacheExecutor.submit(() -> findCachedAttributes(entityId, scope, attributeKeys)), wrappedCachedAttributes -> { - List cachedAttributes = wrappedCachedAttributes.values().stream() - .map(TbCacheValueWrapper::get) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - if (wrappedCachedAttributes.size() == attributeKeys.size()) { - log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); - return Futures.immediateFuture(cachedAttributes); - } - - Set notFoundAttributeKeys = new HashSet<>(attributeKeys); - notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - - List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - - // DB call should run in DB executor, not in cache-related executor - return jpaExecutorService.submit(() -> { - var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); - try { - log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - for (AttributeKvEntry foundInDbAttribute : result) { - AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); - cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); - notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); - } - for (String key : notFoundAttributeKeys) { - cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); - } - List mergedAttributes = new ArrayList<>(cachedAttributes); - mergedAttributes.addAll(result); - cacheTransaction.commit(); - log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); - return mergedAttributes; - } catch (Throwable e) { - cacheTransaction.rollback(); - log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); - throw e; - } - }); - - }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor + List cachedAttributes = wrappedCachedAttributes.values().stream() + .map(TbCacheValueWrapper::get) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (wrappedCachedAttributes.size() == attributeKeys.size()) { + log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); + return Futures.immediateFuture(cachedAttributes); + } + + Set notFoundAttributeKeys = new HashSet<>(attributeKeys); + notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); + + List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); + + // DB call should run in DB executor, not in cache-related executor + return jpaExecutorService.submit(() -> { + var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); + try { + log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + for (AttributeKvEntry foundInDbAttribute : result) { + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); + cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); + } + for (String key : notFoundAttributeKeys) { + cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(result); + cacheTransaction.commit(); + log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); + return mergedAttributes; + } catch (Throwable e) { + cacheTransaction.rollback(); + log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); + throw e; + } + }); + + }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor } private Map> findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { @@ -268,4 +269,15 @@ public class CachedAttributesService implements AttributesService { }, cacheExecutor)).collect(Collectors.toList())); } + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> result = attributesDao.removeAllByEntityId(tenantId, entityId); + result.forEach(deleted -> { + String scope = deleted.getKey(); + String key = deleted.getValue(); + cache.evict(new AttributeCacheKey(scope, entityId, key)); + }); + return result.size(); + } + } From faa93de1027bacffe781611b5dfbc60f25b268c5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 15 Feb 2024 14:26:33 +0200 Subject: [PATCH 06/72] Tenant deletion with housekeeper --- .../DefaultHousekeeperService.java | 7 +++- .../HousekeeperReprocessingService.java | 5 ++- ...ava => EntitiesDeletionTaskProcessor.java} | 18 +++++--- .../server/dao/entity/EntityDaoService.java | 4 ++ .../server/common/data/EntityType.java | 3 +- .../dao/asset/AssetProfileServiceImpl.java | 5 +++ .../server/dao/asset/BaseAssetService.java | 5 +++ .../dao/customer/CustomerServiceImpl.java | 5 +++ .../dao/dashboard/DashboardServiceImpl.java | 5 +++ .../dao/device/DeviceProfileServiceImpl.java | 5 +++ .../server/dao/device/DeviceServiceImpl.java | 6 +++ .../server/dao/edge/EdgeServiceImpl.java | 5 +++ .../dao/entityview/EntityViewServiceImpl.java | 5 +++ .../dao/housekeeper/CleanUpService.java | 9 ++++ .../dao/housekeeper/HousekeeperService.java | 5 +++ .../data/EntitiesDeletionHousekeeperTask.java | 32 +++++++++++++++ .../dao/housekeeper/data/HousekeeperTask.java | 5 +++ .../housekeeper/data/HousekeeperTaskType.java | 2 +- .../DefaultNotificationRequestService.java | 5 +++ .../DefaultNotificationRuleService.java | 5 +++ .../DefaultNotificationTargetService.java | 5 +++ .../DefaultNotificationTemplateService.java | 5 +++ .../server/dao/ota/BaseOtaPackageService.java | 5 +++ .../server/dao/queue/BaseQueueService.java | 5 +++ .../dao/resource/BaseResourceService.java | 5 +++ .../server/dao/rpc/BaseRpcService.java | 5 +++ .../server/dao/rule/BaseRuleChainService.java | 5 +++ .../settings/AdminSettingsServiceImpl.java | 23 ++++++++++- .../dao/sql/attributes/JpaAttributeDao.java | 4 +- .../dao/sql/user/JpaUserSettingsDao.java | 6 +++ .../dao/sql/user/UserSettingsRepository.java | 11 +++++ .../server/dao/tenant/TenantServiceImpl.java | 41 ++++++------------- .../usagerecord/ApiUsageStateServiceImpl.java | 5 +++ .../server/dao/user/UserServiceImpl.java | 20 +++++++++ .../server/dao/user/UserSettingsDao.java | 3 ++ .../dao/widget/WidgetTypeServiceImpl.java | 5 +++ .../dao/widget/WidgetsBundleServiceImpl.java | 5 +++ 37 files changed, 258 insertions(+), 41 deletions(-) rename application/src/main/java/org/thingsboard/server/service/housekeeper/processor/{EntityDeletionTaskProcessor.java => EntitiesDeletionTaskProcessor.java} (53%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index f669df5305..9361baae2b 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -128,8 +128,13 @@ public class DefaultHousekeeperService implements HousekeeperService { @Override public void submitTask(HousekeeperTask task) { + submitTask(UUID.randomUUID(), task); + } + + @Override + public void submitTask(UUID key, HousekeeperTask task) { TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ToHousekeeperServiceMsg.newBuilder() + producer.send(tpi, new TbProtoQueueMsg<>(key, ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) .setTs(task.getTs()) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 4fca14e44f..ac0051def1 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -49,7 +49,7 @@ public class HousekeeperReprocessingService { @Value("${queue.core.housekeeper.poll-interval-ms:10000}") private int pollInterval; - private final long startTs = System.currentTimeMillis(); + private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing private boolean stopped; // todo: stats @@ -114,12 +114,13 @@ public class HousekeeperReprocessingService { msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis()) + .setTs(System.currentTimeMillis()) // maybe set ts + 1 hour so that no-one reprocesses it immediately .build()) .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + // fixme submit with the same msg key, so that the messages goes to this consumer and will not be processed by anyone else producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java similarity index 53% rename from application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java rename to application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index deb693c466..da7af3c42b 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -17,21 +17,29 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.dao.housekeeper.data.EntitiesDeletionHousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EntityDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntitiesDeletionTaskProcessor implements HousekeeperTaskProcessor { - @Override - public void process(HousekeeperTask task) throws Exception { + private final EntityServiceRegistry entityServiceRegistry; + @Override + public void process(EntitiesDeletionHousekeeperTask task) throws Exception { + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(task.getEntityType()); + if (entityService == null) { + throw new IllegalArgumentException("Unsupported entity type " + task.getEntityType()); + } + entityService.deleteByTenantId(task.getTenantId()); } @Override public HousekeeperTaskType getTaskType() { - return HousekeeperTaskType.DELETE_ENTITY; + return HousekeeperTaskType.DELETE_ENTITIES; } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java index b1f6ab9cdb..992c4fad94 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java @@ -34,6 +34,10 @@ public interface EntityDaoService { throw new IllegalArgumentException(getEntityType().getNormalName() + " deletion not supported"); } + default void deleteByTenantId(TenantId tenantId) { + throw new IllegalArgumentException("Deletion by tenant id not supported for " + getEntityType().getNormalName()); + } + EntityType getEntityType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index fb4fe1011e..6316ab1281 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -58,7 +58,8 @@ public enum EntityType { NOTIFICATION_TEMPLATE (30), NOTIFICATION_REQUEST (31), NOTIFICATION (32), - NOTIFICATION_RULE (33); + NOTIFICATION_RULE (33), + ADMIN_SETTINGS(34); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 851e68d3dc..686b75befb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -304,6 +304,11 @@ public class AssetProfileServiceImpl extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findAssetProfileById(tenantId, new AssetProfileId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index c6550f7e25..a57b232ec2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -279,6 +279,11 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findAssetsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 028b6ff59b..bd8ddb4059 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -180,6 +180,11 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom customersByTenantRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteCustomersByTenantId(tenantId); + } + private PaginatedRemover customersByTenantRemover = new PaginatedRemover() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index e08653856b..37abeb993c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -272,6 +272,11 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb tenantDashboardsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteDashboardsByTenantId(tenantId); + } + @Override public PageData findDashboardsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDashboardsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index f9be1c9988..ceebe8f7b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -359,6 +359,11 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findDeviceProfileById(tenantId, new DeviceProfileId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 01894a8cb8..7a8daa6d7c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -422,6 +422,12 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 118449b698..0c95a6f0c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -266,6 +266,11 @@ public class EdgeServiceImpl extends AbstractCachedEntityService findEdgesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findEdgesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 6940b42794..7a5f873a25 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -344,6 +344,11 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService> findEntityViewTypesByTenantId(TenantId tenantId) { log.trace("Executing findEntityViewTypesByTenantId, tenantId [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 0086e17926..1097ec5ece 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; +import java.util.UUID; + @Component @RequiredArgsConstructor @Slf4j @@ -57,4 +59,11 @@ public class CleanUpService { housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); } + public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { + UUID tasksKey = UUID.randomUUID(); // so that all tasks are processed synchronously from one partition + for (EntityType entityType : entityTypes) { + housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java index 85577a2296..5e55318953 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -17,8 +17,13 @@ package org.thingsboard.server.dao.housekeeper; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import java.util.UUID; + public interface HousekeeperService { void submitTask(HousekeeperTask task); + // tasks with the same key will be pushed to the same partition and thus processed synchronously + void submitTask(UUID key, HousekeeperTask task); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java new file mode 100644 index 0000000000..6c87fa9d3c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + + private final EntityType entityType; + + protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { + super(tenantId, null, HousekeeperTaskType.DELETE_ENTITIES); + this.entityType = entityType; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index 9fcbbfe43d..dfcf445d8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; import lombok.Data; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -57,4 +58,8 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ENTITY_ALARMS); } + public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entityType) { + return new EntitiesDeletionHousekeeperTask(tenantId, entityType); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java index 22951280c1..51fb9ab926 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; public enum HousekeeperTaskType { - DELETE_ENTITY, + DELETE_ENTITIES, DELETE_ATTRIBUTES, DELETE_TELEMETRY, // maybe divide into latest and ts kv history? DELETE_EVENTS, diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index 2ad0a59824..9573539297 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -104,6 +104,11 @@ public class DefaultNotificationRequestService implements NotificationRequestSer notificationRequestDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRequestsByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationRequestById(tenantId, new NotificationRequestId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java index 909febe6a8..77a7cd96e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java @@ -93,6 +93,11 @@ public class DefaultNotificationRuleService extends AbstractEntityService implem notificationRuleDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRulesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationRuleById(tenantId, new NotificationRuleId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java index a92e4f6a4e..5996ec6c90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java @@ -201,6 +201,11 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl notificationTargetDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTargetsByTenantId(tenantId); + } + @Override public long countNotificationTargetsByTenantId(TenantId tenantId) { return notificationTargetDao.countByTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index 17b166c3ec..6cf8239f47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -89,6 +89,11 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im notificationTemplateDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTemplatesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationTemplateById(tenantId, new NotificationTemplateId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 53a585c358..e4a4865f02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -230,6 +230,11 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = new PaginatedRemover<>() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index 57c45b1d0b..ebbb69078c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -129,6 +129,11 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ tenantQueuesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteQueuesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findQueueById(tenantId, new QueueId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 35e13bff3a..a80772b506 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -203,6 +203,11 @@ public class BaseResourceService extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findResourceInfoById(tenantId, new TbResourceId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java index 54d7a2170c..9c65d1efe9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java @@ -66,6 +66,11 @@ public class BaseRpcService implements RpcService { tenantRpcRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAllRpcByTenantId(tenantId); + } + @Override public Rpc findById(TenantId tenantId, RpcId rpcId) { log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, rpcId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index ebbec8969f..fe685d61e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -443,6 +443,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC tenantRuleChainsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteRuleChainsByTenantId(tenantId); + } + @Override public RuleChainData exportTenantRuleChains(TenantId tenantId, PageLink pageLink) { Validator.validateId(tenantId, "Incorrect tenant id for search rule chain request."); diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bfc99d1b54..1100d40d22 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -21,14 +21,20 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; +import java.util.Optional; + @Service @Slf4j -public class AdminSettingsServiceImpl implements AdminSettingsService { +public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDaoService { @Autowired private AdminSettingsDao adminSettingsDao; @@ -105,4 +111,19 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { } } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAdminSettingsByTenantId(tenantId); + } + + @Override + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.empty(); + } + + @Override + public EntityType getEntityType() { + return EntityType.ADMIN_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index d8cf6aab32..3730718afa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -196,8 +196,8 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_type = ? and entity_id = ? " + "RETURNING attribute_type, attribute_key", entityId.getEntityType().name(), entityId.getId()).stream() - .map(deleted -> Pair.of((String) deleted.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), - (String) deleted.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) + .map(row -> Pair.of((String) row.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), + (String) row.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) .collect(Collectors.toList()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index 1feabaacb5..914806e829 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.dao.DaoUtil; @@ -50,4 +51,9 @@ public class JpaUserSettingsDao extends JpaAbstractDaoListeningExecutorService i userSettingsRepository.deleteById(id); } + @Override + public void removeByUserId(TenantId tenantId, UserId userId) { + userSettingsRepository.deleteByUserId(userId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index cb81675a80..0a8d3e431a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -16,9 +16,20 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; +import java.util.UUID; + public interface UserSettingsRepository extends JpaRepository { + @Transactional + @Modifying + @Query("DELETE FROM UserSettingsEntity s WHERE s.userId = :userId") + void deleteByUserId(@Param("userId") UUID userId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 6fb2a93d54..0ca5e76a93 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -211,41 +211,26 @@ public class TenantServiceImpl extends AbstractCachedEntityService userValidator; private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; @@ -255,6 +256,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic validateId(userId, INCORRECT_USER_ID + userId); userCredentialsDao.removeByUserId(tenantId, userId); userAuthSettingsDao.removeByUserId(userId); + userSettingsDao.removeByUserId(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); @@ -313,6 +315,12 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic tenantAdminsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + log.trace("Executing deleteByTenantId, tenantId [{}]", tenantId); + usersRemover.removeEntities(tenantId, tenantId); + } + @Override public PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findCustomerUsers, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -464,6 +472,18 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic } }; + private final PaginatedRemover usersRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return findUsersByTenantId(tenantId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, User user) { + deleteUser(tenantId, user); + } + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findUserById(tenantId, new UserId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java index f2e5474c58..aac0b8f4de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.user; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; @@ -27,4 +28,6 @@ public interface UserSettingsDao { void removeById(TenantId tenantId, UserSettingsCompositeKey key); + void removeByUserId(TenantId tenantId, UserId userId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 7f7d4d5ae8..aad37a8f66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -226,6 +226,11 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { tenantWidgetTypeRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetTypesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findWidgetTypeById(tenantId, new WidgetTypeId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 9d6960a7d1..1631bfbdbf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -179,6 +179,11 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { tenantWidgetsBundleRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetsBundlesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId()))); From 4cbd57c923f2509fe9449c1bff21c06fdf698977 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 16 Feb 2024 13:34:26 +0200 Subject: [PATCH 07/72] Telemetry deletion task; task processing timeout --- .../DefaultHousekeeperService.java | 37 ++++++++++++++----- .../HousekeeperReprocessingService.java | 31 +++++----------- .../TelemetryDeletionTaskProcessor.java | 14 ++++++- .../dao/attributes/BaseAttributesService.java | 1 + .../data/EntitiesDeletionHousekeeperTask.java | 2 +- .../dao/housekeeper/data/HousekeeperTask.java | 3 +- 6 files changed, 54 insertions(+), 34 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 9361baae2b..10d53f3500 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -42,8 +42,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @TbCoreComponent @@ -57,10 +61,13 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; private final DataDecodingEncodingService dataDecodingEncodingService; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @Value("${queue.core.housekeeper.poll-interval-ms:10000}") private int pollInterval; + private int taskProcessingTimeout = 120; private boolean stopped; @@ -89,9 +96,10 @@ public class DefaultHousekeeperService implements HousekeeperService { for (TbProtoQueueMsg msg : msgs) { try { - processTask(msg.getValue()); - } catch (Exception e) { - log.error("Message processing failed", e); + processTask(msg); + } catch (Throwable e) { + log.error("Unexpected error during message processing [{}]", msg, e); + reprocessingService.submitForReprocessing(msg); } } consumer.commit(); @@ -111,18 +119,29 @@ public class DefaultHousekeeperService implements HousekeeperService { } @SuppressWarnings("unchecked") - protected void processTask(ToHousekeeperServiceMsg msg) { + protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { + ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); - log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); try { - taskProcessor.process((T) task); - } catch (Exception e) { + Future future = executor.submit(() -> { + taskProcessor.process((T) task); + return null; + }); + future.get(taskProcessingTimeout, TimeUnit.SECONDS); + } catch (ExecutionException executionException) { + Throwable error = executionException.getCause(); log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - task.getTaskType(), msg.getTask().getAttempt(), task, e); - reprocessingService.submitForReprocessing(msg); + task.getTaskType(), msg.getTask().getAttempt(), task, error); + reprocessingService.submitForReprocessing(queueMsg); + } catch (TimeoutException timeoutException) { + log.error("[{}][{}][{}] {} task processing timeout after {} seconds, submitting for reprocessing (attempt {}): {}", + task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + task.getTaskType(), taskProcessingTimeout, msg.getTask().getAttempt(), task); + reprocessingService.submitForReprocessing(queueMsg); // fixme: we should schedule such task for later (separate queue?) } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index ac0051def1..9d7114476b 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -32,9 +32,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PreDestroy; import java.util.List; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; @TbCoreComponent @Service @@ -46,7 +46,7 @@ public class HousekeeperReprocessingService { private final TbQueueConsumer> consumer; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing @@ -68,28 +68,17 @@ public class HousekeeperReprocessingService { while (!stopped && !consumer.isStopped()) { try { List> msgs = consumer.poll(pollInterval); - if (msgs.isEmpty()) { + if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately stop(); return; } - if (msgs.stream().anyMatch(msg -> { - boolean newMsg = msg.getValue().getTask().getTs() >= startTs; - if (newMsg) { - log.info("Stopping reprocessing due to msg is new {}", msg); - } - return newMsg; - })) { - stop(); // fixme: we should commit already reprocessed messages; maybe submit for reprocessing again and commit? - // msg batch size should be 1. otherwise some tasks won't be reprocessed - return; - } for (TbProtoQueueMsg msg : msgs) { try { - housekeeperService.processTask(msg.getValue());// fixme: or should we submit to queue? - Thread.sleep(1000); + housekeeperService.processTask(msg); } catch (Exception e) { - log.error("Message processing failed", e); + log.error("Unexpected error during message reprocessing [{}]", msg, e); + submitForReprocessing(msg); } } consumer.commit(); @@ -108,20 +97,20 @@ public class HousekeeperReprocessingService { log.info("Started Housekeeper tasks reprocessing"); } - public void submitForReprocessing(ToHousekeeperServiceMsg msg) { + public void submitForReprocessing(TbProtoQueueMsg queueMsg) { + ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis()) // maybe set ts + 1 hour so that no-one reprocesses it immediately + .setTs(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)) // so that it is not reprocessed by anyone in the nearest hour .build()) .build(); var producer = producerProvider.getHousekeeperDelayedMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - // fixme submit with the same msg key, so that the messages goes to this consumer and will not be processed by anyone else - producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index e925c9f20c..7159c43f46 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -16,21 +16,31 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import java.util.List; + @Component @RequiredArgsConstructor +@Slf4j public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; @Override public void process(HousekeeperTask task) throws Exception { -// timeseriesService.remove() -// timeseriesService.removeAllLatest(); + List keys = timeseriesService.findAllKeysByEntityIds(task.getTenantId(), List.of(task.getEntityId())); + for (String key : keys) { + DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); + timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); + } + log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 1cace5b71c..9359383071 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -118,6 +118,7 @@ public class BaseAttributesService implements AttributesService { @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + // todo: probably better to retrieve by scope in case there are too many attributes return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 6c87fa9d3c..8f4a8d38b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -25,7 +25,7 @@ public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private final EntityType entityType; protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { - super(tenantId, null, HousekeeperTaskType.DELETE_ENTITIES); + super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); this.entityType = entityType; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index dfcf445d8e..f9389e4566 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.housekeeper.data; import lombok.Data; +import lombok.NonNull; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; @@ -31,7 +32,7 @@ public class HousekeeperTask implements Serializable { private final HousekeeperTaskType taskType; private final long ts; - protected HousekeeperTask(TenantId tenantId, EntityId entityId, HousekeeperTaskType taskType) { + protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) { this.tenantId = tenantId; this.entityId = entityId; this.taskType = taskType; From 721854c8793e5f89c71cfdf05fb2c2320ed6a62b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 20 Feb 2024 18:50:29 +0200 Subject: [PATCH 08/72] Housekeeper: reprocessing improvements --- .../DefaultHousekeeperService.java | 20 ++--- .../HousekeeperReprocessingService.java | 52 ++++++++---- .../EventsDeletionTaskProcessor.java | 1 + common/proto/src/main/proto/queue.proto | 83 ++++++++++--------- .../provider/AwsSqsMonolithQueueFactory.java | 4 +- .../provider/AwsSqsTbCoreQueueFactory.java | 4 +- .../InMemoryMonolithQueueFactory.java | 8 +- .../provider/KafkaMonolithQueueFactory.java | 14 ++-- .../provider/KafkaTbCoreQueueFactory.java | 14 ++-- .../provider/PubSubMonolithQueueFactory.java | 4 +- .../provider/PubSubTbCoreQueueFactory.java | 4 +- .../RabbitMqMonolithQueueFactory.java | 4 +- .../provider/RabbitMqTbCoreQueueFactory.java | 4 +- .../ServiceBusMonolithQueueFactory.java | 4 +- .../ServiceBusTbCoreQueueFactory.java | 4 +- .../queue/provider/TbCoreQueueFactory.java | 5 +- .../provider/TbCoreQueueProducerProvider.java | 9 +- .../provider/TbQueueProducerProvider.java | 2 +- .../TbRuleEngineProducerProvider.java | 2 +- .../TbTransportQueueProducerProvider.java | 2 +- .../TbVersionControlProducerProvider.java | 2 +- .../queue/settings/TbQueueCoreSettings.java | 2 +- 22 files changed, 134 insertions(+), 114 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 10d53f3500..644e3cc2c7 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -99,7 +99,7 @@ public class DefaultHousekeeperService implements HousekeeperService { processTask(msg); } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); - reprocessingService.submitForReprocessing(msg); + reprocessingService.submitForReprocessing(msg, e); } } consumer.commit(); @@ -131,23 +131,23 @@ public class DefaultHousekeeperService implements HousekeeperService { return null; }); future.get(taskProcessingTimeout, TimeUnit.SECONDS); - } catch (ExecutionException executionException) { - Throwable error = executionException.getCause(); + } catch (Throwable e) { + Throwable error = e; + if (e instanceof ExecutionException) { + error = error.getCause(); + } else if (e instanceof TimeoutException) { + error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds"); + } log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType(), msg.getTask().getAttempt(), task, error); - reprocessingService.submitForReprocessing(queueMsg); - } catch (TimeoutException timeoutException) { - log.error("[{}][{}][{}] {} task processing timeout after {} seconds, submitting for reprocessing (attempt {}): {}", - task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - task.getTaskType(), taskProcessingTimeout, msg.getTask().getAttempt(), task); - reprocessingService.submitForReprocessing(queueMsg); // fixme: we should schedule such task for later (separate queue?) + reprocessingService.submitForReprocessing(queueMsg, error); } } @Override public void submitTask(HousekeeperTask task) { - submitTask(UUID.randomUUID(), task); + submitTask(task.getEntityId().getId(), task); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 9d7114476b..2d2c3242f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -16,18 +16,23 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PreDestroy; @@ -42,35 +47,46 @@ import java.util.concurrent.TimeUnit; public class HousekeeperReprocessingService { private final DefaultHousekeeperService housekeeperService; + private final PartitionService partitionService; + private final TbCoreQueueFactory queueFactory; private final TbQueueProducerProvider producerProvider; - private final TbQueueConsumer> consumer; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + private static final int startDelay = 15; // fixme - to 5 minutes + private static final int reprocessingDelay = 30; // seconds + private static final int maxReprocessingAttempts = 5; @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing private boolean stopped; // todo: stats public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, - TbCoreQueueFactory queueFactory, + PartitionService partitionService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider) { this.housekeeperService = housekeeperService; - this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer(); + this.partitionService = partitionService; + this.queueFactory = queueFactory; this.producerProvider = producerProvider; } - @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) - public void afterStartUp() { + @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS) + public void startReprocessing() { + if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { + return; + } + + var consumer = queueFactory.createHousekeeperReprocessingMsgConsumer(); consumer.subscribe(); consumerExecutor.submit(() -> { - while (!stopped && !consumer.isStopped()) { + log.info("Starting Housekeeper tasks reprocessing"); + long startTs = System.currentTimeMillis(); + while (!stopped) { try { List> msgs = consumer.poll(pollInterval); if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately - stop(); - return; + break; } for (TbProtoQueueMsg msg : msgs) { @@ -78,7 +94,7 @@ public class HousekeeperReprocessingService { housekeeperService.processTask(msg); } catch (Exception e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); - submitForReprocessing(msg); + submitForReprocessing(msg, e); } } consumer.commit(); @@ -93,31 +109,33 @@ public class HousekeeperReprocessingService { } } } + consumer.unsubscribe(); + log.info("Stopped Housekeeper tasks reprocessing"); }); - log.info("Started Housekeeper tasks reprocessing"); } - public void submitForReprocessing(TbProtoQueueMsg queueMsg) { + // todo: dead letter queue if attempts count exceeds the configured maximum + public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .setTs(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)) // so that it is not reprocessed by anyone in the nearest hour + .addErrors(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)) + .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(reprocessingDelay)) .build()) .build(); - var producer = producerProvider.getHousekeeperDelayedMsgProducer(); + log.trace("Submitting for reprocessing: {}", msg); + var producer = producerProvider.getHousekeeperReprocessingMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); } @PreDestroy private void stop() { - log.info("Stopped Housekeeper tasks reprocessing"); stopped = true; - consumer.unsubscribe(); consumerExecutor.shutdownNow(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 56edf01637..b95d7fc1e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -29,6 +29,7 @@ public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index a1be25f29d..1a4f92fcd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -217,12 +217,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 1ee8b404a5..310bc75fe4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -171,13 +171,13 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { - return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { - return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())); + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 5ee9b6f53b..b4c2220ac1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -373,22 +373,22 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) - .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) - .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) .admin(housekeeperAdmin) .build(); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) - .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer")) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index c01ba32f70..635eb3e300 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -330,22 +330,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) - .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId()) - .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) + .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) .admin(housekeeperAdmin) .build(); } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) - .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic())) - .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId()) - .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer")) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index a63a9fc16b..e6a23b8068 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -239,12 +239,12 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 5f1f282e39..1b52a2ba42 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -210,12 +210,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 802b08773d..fa89956b64 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -236,12 +236,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index ceb6e03ca5..b1f3de672f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -209,12 +209,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 1e9c1cadaf..5f4be98ca0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -235,12 +235,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e39df45b82..579fba272c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -210,12 +210,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperDelayedMsgProducer() { + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { return null; } @Override - public TbQueueConsumer> createHousekeeperDelayedMsgConsumer() { + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { return null; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 279a469415..866eb931f2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -137,8 +136,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueConsumer> createHousekeeperMsgConsumer(); - TbQueueProducer> createHousekeeperDelayedMsgProducer(); + TbQueueProducer> createHousekeeperReprocessingMsgProducer(); - TbQueueConsumer> createHousekeeperDelayedMsgConsumer(); + TbQueueConsumer> createHousekeeperReprocessingMsgConsumer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 4fa22cb7e3..e6bc8f0484 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -44,7 +43,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toHousekeeperDelayed; + private TbQueueProducer> toHousekeeperReprocessing; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -60,7 +59,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); - this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer(); + this.toHousekeeperReprocessing = tbQueueProvider.createHousekeeperReprocessingMsgProducer(); } @Override @@ -104,8 +103,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { - return toHousekeeperDelayed; + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { + return toHousekeeperReprocessing; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 5cdf390f47..ff56d2e8be 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -82,6 +82,6 @@ public interface TbQueueProducerProvider { TbQueueProducer> getHousekeeperMsgProducer(); - TbQueueProducer> getHousekeeperDelayedMsgProducer(); + TbQueueProducer> getHousekeeperReprocessingMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index c5ddf57fc0..ee6700906c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -96,7 +96,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index e8dd7431a6..869c588fc7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -92,7 +92,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index 7ddd2109ba..3f4bd209e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -88,7 +88,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider } @Override - public TbQueueProducer> getHousekeeperDelayedMsgProducer() { + public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0b66c2fa75..2aa01193b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -38,7 +38,7 @@ public class TbQueueCoreSettings { private String housekeeperTopic; @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}") - private String housekeeperDelayedTopic; + private String housekeeperReprocessingTopic; @Value("${queue.core.partitions}") private int partitions; From b72af4ead902737e490ae93cdb5696e15ca84e9a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 21 Feb 2024 18:49:15 +0200 Subject: [PATCH 09/72] Housekeeper tasks encoded to json --- .../DefaultHousekeeperService.java | 26 +++++++++---------- .../HousekeeperReprocessingService.java | 18 ++++++++++--- .../src/main/resources/thingsboard.yml | 4 +++ .../data/notification/NotificationType.java | 1 + common/proto/src/main/proto/queue.proto | 2 +- .../queue/kafka/TbKafkaTopicConfigs.java | 5 ++++ .../provider/KafkaMonolithQueueFactory.java | 6 +++-- .../provider/KafkaTbCoreQueueFactory.java | 6 +++-- .../data/AlarmsUnassignHousekeeperTask.java | 8 +++--- .../data/EntitiesDeletionHousekeeperTask.java | 8 +++--- .../dao/housekeeper/data/HousekeeperTask.java | 21 ++++++++++++--- 11 files changed, 73 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 644e3cc2c7..76bf40fc36 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -15,11 +15,11 @@ */ package org.thingsboard.server.service.housekeeper; -import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.housekeeper.HousekeeperService; @@ -40,7 +40,6 @@ import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProce import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -95,8 +94,11 @@ public class DefaultHousekeeperService implements HousekeeperService { } for (TbProtoQueueMsg msg : msgs) { + log.trace("Processing task: {}", msg); try { processTask(msg); + } catch (InterruptedException e) { + return; } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); reprocessingService.submitForReprocessing(msg, e); @@ -121,16 +123,20 @@ public class DefaultHousekeeperService implements HousekeeperService { @SuppressWarnings("unchecked") protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { ToHousekeeperServiceMsg msg = queueMsg.getValue(); - HousekeeperTask task = dataDecodingEncodingService.decode(msg.getTask().getValue().toByteArray()).get(); - HousekeeperTaskProcessor taskProcessor = getTaskProcessor(task.getTaskType()); + HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); + HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(task.getTaskType()); + if (taskProcessor == null) { + throw new IllegalArgumentException("Unsupported task type " + task.getTaskType()); + } - log.info("[{}][{}][{}] Processing task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); try { Future future = executor.submit(() -> { taskProcessor.process((T) task); return null; }); future.get(taskProcessingTimeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw e; } catch (Throwable e) { Throwable error = e; if (e instanceof ExecutionException) { @@ -152,21 +158,15 @@ public class DefaultHousekeeperService implements HousekeeperService { @Override public void submitTask(UUID key, HousekeeperTask task) { + log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); producer.send(tpi, new TbProtoQueueMsg<>(key, ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() - .setValue(ByteString.copyFrom(dataDecodingEncodingService.encode(task))) + .setValue(JacksonUtil.toString(task)) .setTs(task.getTs()) .setAttempt(0) .build()) .build()), null); - log.trace("[{}][{}][{}] Submitted task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); - } - - @SuppressWarnings("unchecked") - private HousekeeperTaskProcessor getTaskProcessor(HousekeeperTaskType taskType) { - return Optional.ofNullable((HousekeeperTaskProcessor) taskProcessors.get(taskType)) - .orElseThrow(() -> new IllegalArgumentException("Unsupported task type " + taskType)); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 2d2c3242f9..44b2728089 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -28,7 +28,8 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; @@ -36,7 +37,9 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PreDestroy; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -90,11 +93,15 @@ public class HousekeeperReprocessingService { } for (TbProtoQueueMsg msg : msgs) { + log.trace("Reprocessing task: {}", msg); try { housekeeperService.processTask(msg); - } catch (Exception e) { + } catch (InterruptedException e) { + return; + } catch (Throwable e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); submitForReprocessing(msg, e); + // fixme: msgs are duplicated } } consumer.commit(); @@ -118,12 +125,15 @@ public class HousekeeperReprocessingService { public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { ToHousekeeperServiceMsg msg = queueMsg.getValue(); HousekeeperTaskProto task = msg.getTask(); + int attempt = task.getAttempt() + 1; + Set errors = new LinkedHashSet<>(task.getErrorsList()); + errors.add(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)); msg = msg.toBuilder() .setTask(task.toBuilder() .setAttempt(attempt) - .addErrors(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)) - .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(reprocessingDelay)) + .clearErrors().addAllErrors(errors) + .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis((long) (reprocessingDelay * 0.8))) .build()) .build(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7af241d41e..f59e3c86b7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1403,6 +1403,9 @@ queue: tb_housekeeper: - key: max.poll.records value: "1" + tb_housekeeper.reprocessing: + - key: max.poll.records + value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1425,6 +1428,7 @@ queue: # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" + housekeeper-reprocessing: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java index cfc91d8fca..5a8fd87933 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java @@ -31,4 +31,5 @@ public enum NotificationType { RATE_LIMITS, EDGE_CONNECTION, EDGE_COMMUNICATION_FAILURE + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 73cea54a47..8e757a3b14 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1424,7 +1424,7 @@ message ToHousekeeperServiceMsg { } message HousekeeperTaskProto { - bytes value = 1; + string value = 1; int64 ts = 2; // reprocessing diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index d3a5eb821d..915f61c8b4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -45,6 +45,8 @@ public class TbKafkaTopicConfigs { private String vcProperties; @Value("${queue.kafka.topic-properties.housekeeper:}") private String housekeeperProperties; + @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") + private String housekeeperReprocessingProperties; @Getter private Map coreConfigs; @@ -66,6 +68,8 @@ public class TbKafkaTopicConfigs { private Map vcConfigs; @Getter private Map housekeeperConfigs; + @Getter + private Map housekeeperReprocessingConfigs; @PostConstruct private void init() { @@ -81,6 +85,7 @@ public class TbKafkaTopicConfigs { fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); + housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index c8a11e1f32..95bf2d1195 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -84,6 +84,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -118,6 +119,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -378,7 +380,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi .settings(kafkaSettings) .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .build(); } @@ -390,7 +392,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .statsService(consumerStatsService) .build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index f2812564e5..478963d300 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -82,6 +82,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -115,6 +116,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -335,7 +337,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { .settings(kafkaSettings) .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .build(); } @@ -347,7 +349,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) - .admin(housekeeperAdmin) + .admin(housekeeperReprocessingAdmin) .statsService(consumerStatsService) .build(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index aa1858f59f..aea09c093c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -15,13 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.User; -@Getter +@Data +@EqualsAndHashCode(callSuper = true) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { - private final String userTitle; + private String userTitle; protected AlarmsUnassignHousekeeperTask(User user) { super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 8f4a8d38b0..7846feb229 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -15,14 +15,16 @@ */ package org.thingsboard.server.dao.housekeeper.data; -import lombok.Getter; +import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; -@Getter +@Data +@EqualsAndHashCode(callSuper = true) public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { - private final EntityType entityType; + private EntityType entityType; protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java index f9389e4566..6b63d43d43 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java @@ -15,7 +15,13 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AccessLevel; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.NonNull; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -24,13 +30,20 @@ import org.thingsboard.server.common.data.id.TenantId; import java.io.Serializable; +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "taskType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = HousekeeperTask.class) +@JsonSubTypes({ + @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), + @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) +}) @Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class HousekeeperTask implements Serializable { - private final TenantId tenantId; - private final EntityId entityId; - private final HousekeeperTaskType taskType; - private final long ts; + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private long ts; protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) { this.tenantId = tenantId; From 7cc599b5b59ab0f65b78049e7c5b2db729618c9a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 13:58:15 +0200 Subject: [PATCH 10/72] Housekeeper stats; Grafana dashboard --- .../DefaultHousekeeperService.java | 23 +- .../HousekeeperReprocessingService.java | 33 +- .../AlarmsUnassignTaskProcessor.java | 2 +- .../AttributesDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 3 +- .../TelemetryDeletionTaskProcessor.java | 2 +- .../stats/HousekeeperStatsService.java | 117 ++++++ .../src/main/resources/thingsboard.yml | 11 +- .../common/stats/DefaultStatsFactory.java | 20 +- .../server/common/stats/StatsFactory.java | 5 + .../server/common/stats/StatsType.java | 7 +- .../provisioning/dashboards/housekeeper.json | 390 ++++++++++++++++++ 12 files changed, 588 insertions(+), 27 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java create mode 100644 docker/monitoring/grafana/provisioning/dashboards/housekeeper.json diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 76bf40fc36..a324f279fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -33,9 +33,9 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; +import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import javax.annotation.PreDestroy; import java.util.List; @@ -59,27 +59,28 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueConsumer> consumer; private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; - private final DataDecodingEncodingService dataDecodingEncodingService; + private final HousekeeperStatsService statsService; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); - @Value("${queue.core.housekeeper.poll-interval-ms:10000}") + @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") + private int taskProcessingTimeout; + @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - private int taskProcessingTimeout = 120; private boolean stopped; public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, - DataDecodingEncodingService dataDecodingEncodingService, + HousekeeperStatsService statsService, @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; + this.statsService = statsService; this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); - this.dataDecodingEncodingService = dataDecodingEncodingService; } @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) @@ -129,18 +130,23 @@ public class DefaultHousekeeperService implements HousekeeperService { throw new IllegalArgumentException("Unsupported task type " + task.getTaskType()); } + if (log.isDebugEnabled()) { + log.debug("[{}][{}][{}] {} task {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), + msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", task.getTaskType()); + } try { Future future = executor.submit(() -> { taskProcessor.process((T) task); return null; }); - future.get(taskProcessingTimeout, TimeUnit.SECONDS); + future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); + statsService.reportProcessed(task, msg); } catch (InterruptedException e) { throw e; } catch (Throwable e) { Throwable error = e; if (e instanceof ExecutionException) { - error = error.getCause(); + error = e.getCause(); } else if (e instanceof TimeoutException) { error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds"); } @@ -148,6 +154,7 @@ public class DefaultHousekeeperService implements HousekeeperService { task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(queueMsg, error); + statsService.reportFailure(task, msg); } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 44b2728089..de88b2dcea 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.StringUtils; @@ -28,20 +27,20 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.queue.TbQueueCallback; -import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @TbCoreComponent @@ -54,16 +53,19 @@ public class HousekeeperReprocessingService { private final TbCoreQueueFactory queueFactory; private final TbQueueProducerProvider producerProvider; - private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - - private static final int startDelay = 15; // fixme - to 5 minutes - private static final int reprocessingDelay = 30; // seconds - private static final int maxReprocessingAttempts = 5; + @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:15}") // fixme: to 5 minutes + private int startDelay; + @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:30}") // fixme: to 30 minutes or 1 hour + private int reprocessingDelay; + @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") + private int maxReprocessingAttempts; @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler")); + private boolean stopped; - // todo: stats public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, PartitionService partitionService, TbCoreQueueFactory queueFactory, @@ -74,7 +76,17 @@ public class HousekeeperReprocessingService { this.producerProvider = producerProvider; } - @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS) + @PostConstruct + private void init() { + scheduler.scheduleWithFixedDelay(() -> { + try { + startReprocessing(); + } catch (Throwable e) { + log.error("Unexpected error during reprocessing", e); + } + }, startDelay, reprocessingDelay, TimeUnit.SECONDS); + } + public void startReprocessing() { if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { return; @@ -146,6 +158,7 @@ public class HousekeeperReprocessingService { @PreDestroy private void stop() { stopped = true; + scheduler.shutdownNow(); consumerExecutor.shutdownNow(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 247466af65..68fd05deec 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -36,7 +36,7 @@ public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); - log.trace("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); + log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index e3bac35e9b..0467e50aeb 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -32,7 +32,7 @@ public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor @Override public void process(HousekeeperTask task) throws Exception { int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); - log.trace("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index b95d7fc1e3..481f9f1bbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -24,16 +24,17 @@ import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; @Component @RequiredArgsConstructor public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { + private final EventService eventService; @Override public void process(HousekeeperTask task) throws Exception { eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); - throw new RuntimeException("test error"); } @Override public HousekeeperTaskType getTaskType() { return HousekeeperTaskType.DELETE_EVENTS; } + } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index 7159c43f46..e7da740261 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -40,7 +40,7 @@ public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor< DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); } - log.trace("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); + log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java new file mode 100644 index 0000000000..4fcc13837d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.stats; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class HousekeeperStatsService { + + private final Map stats = new EnumMap<>(HousekeeperTaskType.class); + + public HousekeeperStatsService(StatsFactory statsFactory) { + for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) { + stats.put(taskType, new HousekeeperStats(taskType, statsFactory)); + } + } + + @Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS) + private void reportStats() { + String statsStr = stats.values().stream().map(stats -> { + String countersStr = stats.getCounters().stream() + .filter(counter -> counter.get() > 0) + .map(counter -> counter.getName() + " = " + counter.get()) + .collect(Collectors.joining(", ")); + if (countersStr.isEmpty()) { + return null; + } else { + return stats.getTaskType() + " {" + countersStr + "}"; + } + }).filter(Objects::nonNull).collect(Collectors.joining("; ")); + + if (!statsStr.isEmpty()) { + stats.values().forEach(HousekeeperStats::reset); + log.info("Housekeeper stats: {}", statsStr); + } + } + + public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getProcessedCounter().increment(); + } else { + stats.getReprocessedCounter().increment(); + } + } + + public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(task.getTaskType()); + if (msg.getTask().getErrorsCount() == 0) { + stats.getFailedProcessingCounter().increment(); + } else { + stats.getFailedReprocessingCounter().increment(); + } + } + + @Getter + static class HousekeeperStats { + private final HousekeeperTaskType taskType; + private final List counters = new ArrayList<>(); + + private final StatsCounter processedCounter; + private final StatsCounter failedProcessingCounter; + private final StatsCounter reprocessedCounter; + private final StatsCounter failedReprocessingCounter; + + public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) { + this.taskType = taskType; + this.processedCounter = register("processed", statsFactory); + this.failedProcessingCounter = register("failedProcessing", statsFactory); + this.reprocessedCounter = register("reprocessed", statsFactory); + this.failedReprocessingCounter = register("failedReprocessing", statsFactory); + } + + private StatsCounter register(String statsName, StatsFactory statsFactory) { + StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name())); + counters.add(counter); + return counter; + } + + public void reset() { + counters.forEach(DefaultCounter::clear); + } + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f59e3c86b7..4ced0655f0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1585,9 +1585,14 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: - topic: "tb_housekeeper" - reprocessing-topic: "tb_housekeeper.reprocessing" - poll-interval-ms: "1000" + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes + task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}" + vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index a6ec01d089..459f2e0a68 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -25,6 +25,10 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -62,10 +66,24 @@ public class DefaultStatsFactory implements StatsFactory { @Override public StatsCounter createStatsCounter(String key, String statsName) { + return createStatsCounter(key, statsName, Collections.emptyMap()); + } + + @Override + public StatsCounter createStatsCounter(String key, String statsName, Map tags) { + String[] tagsArr = new String[]{STATS_NAME_TAG, statsName}; + if (!tags.isEmpty()) { + List tagsList = new ArrayList<>(List.of(tagsArr)); + tags.forEach((name, value) -> { + tagsList.add(name); + tagsList.add(value); + }); + tagsArr = tagsList.toArray(String[]::new); + } return new StatsCounter( new AtomicInteger(0), metricsEnabled ? - meterRegistry.counter(key, STATS_NAME_TAG, statsName) + meterRegistry.counter(key, tagsArr) : STUB_COUNTER, statsName ); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index 8e6989016b..90a4191ee2 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -17,9 +17,14 @@ package org.thingsboard.server.common.stats; import io.micrometer.core.instrument.Timer; +import java.util.Map; + public interface StatsFactory { + StatsCounter createStatsCounter(String key, String statsName); + StatsCounter createStatsCounter(String key, String statsName, Map tags); + DefaultCounter createDefaultCounter(String key, String... tags); T createGauge(String key, T number, String... tags); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 318f0418de..6c854ac8d6 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -16,7 +16,12 @@ package org.thingsboard.server.common.stats; public enum StatsType { - RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"); + RULE_ENGINE("ruleEngine"), + CORE("core"), + TRANSPORT("transport"), + JS_INVOKE("jsInvoke"), + RATE_EXECUTOR("rateExecutor"), + HOUSEKEEPER("housekeeper"); private String name; diff --git a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json new file mode 100644 index 0000000000..8c68cc9428 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json @@ -0,0 +1,390 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Processing failures", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessed", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))", + "interval": "", + "legendFormat": "{{taskType}}", + "refId": "A" + } + ], + "title": "Reprocessing failures", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Housekeeper", + "uid": "JFJb7voIz", + "version": 5, + "weekStart": "" +} \ No newline at end of file From 85f54200195d4b8e8dca78aecb4b77492c15aa14 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 14:03:16 +0200 Subject: [PATCH 11/72] Revert proto formatting --- common/proto/src/main/proto/queue.proto | 78 ++++++++++++------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8e757a3b14..3ac8c90a1d 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -24,36 +24,36 @@ option java_outer_classname = "TransportProtos"; * Common data structures */ enum EntityTypeProto { - UNSPECIFIED = 0; - TENANT = 1; - CUSTOMER = 2; - USER = 3; - DASHBOARD = 4; - ASSET = 5; - DEVICE = 6; - ALARM = 7; + UNSPECIFIED = 0; + TENANT = 1; + CUSTOMER = 2; + USER = 3; + DASHBOARD = 4; + ASSET = 5; + DEVICE = 6; + ALARM = 7; // next 3 reserved for PE; - RULE_CHAIN = 11; - RULE_NODE = 12; + RULE_CHAIN = 11; + RULE_NODE = 12; // next 2 reserved for PE; - ENTITY_VIEW = 15; - WIDGETS_BUNDLE = 16; - WIDGET_TYPE = 17; + ENTITY_VIEW = 15; + WIDGETS_BUNDLE = 16; + WIDGET_TYPE = 17; // next 2 reserved for PE; - TENANT_PROFILE = 20; - DEVICE_PROFILE = 21; - ASSET_PROFILE = 22; - API_USAGE_STATE = 23; - TB_RESOURCE = 24; - OTA_PACKAGE = 25; - EDGE = 26; - RPC = 27; - QUEUE = 28; - NOTIFICATION_TARGET = 29; + TENANT_PROFILE = 20; + DEVICE_PROFILE = 21; + ASSET_PROFILE = 22; + API_USAGE_STATE = 23; + TB_RESOURCE = 24; + OTA_PACKAGE = 25; + EDGE = 26; + RPC = 27; + QUEUE = 28; + NOTIFICATION_TARGET = 29; NOTIFICATION_TEMPLATE = 30; - NOTIFICATION_REQUEST = 31; - NOTIFICATION = 32; - NOTIFICATION_RULE = 33; + NOTIFICATION_REQUEST = 31; + NOTIFICATION = 32; + NOTIFICATION_RULE = 33; } /** @@ -133,9 +133,9 @@ message KeyValueProto { } enum AttributeScopeProto { - CLIENT_SCOPE = 0; - SERVER_SCOPE = 1; - SHARED_SCOPE = 2; + CLIENT_SCOPE = 0; + SERVER_SCOPE = 1; + SHARED_SCOPE = 2; } message AttributeKey { @@ -259,7 +259,7 @@ message GetOrCreateDeviceFromGatewayResponseMsg { } enum TransportApiRequestErrorCode { - UNKNOWN_TRANSPORT_API_ERROR = 0; + UNKNOWN_TRANSPORT_API_ERROR = 0; ENTITY_LIMIT = 1; } @@ -918,15 +918,15 @@ message FromDeviceRPCResponseProto { } enum ComponentLifecycleEvent { - CREATED = 0; - STARTED = 1; - ACTIVATED = 2; - SUSPENDED = 3; - UPDATED = 4; - STOPPED = 5; - DELETED = 6; - FAILED = 7; - DEACTIVATED = 8; + CREATED = 0; + STARTED = 1; + ACTIVATED = 2; + SUSPENDED = 3; + UPDATED = 4; + STOPPED = 5; + DELETED = 6; + FAILED = 7; + DEACTIVATED = 8; } message ComponentLifecycleMsgProto { From 886c448d61afc621da9b91c32139cf14c3e5725f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 22 Feb 2024 14:57:40 +0200 Subject: [PATCH 12/72] Clean up tenant service --- .../DefaultHousekeeperService.java | 18 +-- .../HousekeeperReprocessingService.java | 10 +- .../stats/HousekeeperStatsService.java | 8 +- ...mponentLifecycleEventTriggerProcessor.java | 2 +- .../server/common/data/EntityType.java | 3 +- .../dao/device/DeviceProfileServiceImpl.java | 6 - .../dao/housekeeper/CleanUpService.java | 8 +- .../data/AlarmsUnassignHousekeeperTask.java | 3 + .../data/EntitiesDeletionHousekeeperTask.java | 3 + .../validator/AssetProfileDataValidator.java | 1 + .../validator/DashboardDataValidator.java | 2 + .../validator/DeviceDataValidator.java | 2 + .../validator/DeviceProfileDataValidator.java | 1 + .../validator/ResourceDataValidator.java | 1 + .../validator/RuleChainDataValidator.java | 1 + .../settings/AdminSettingsServiceImpl.java | 39 ++---- .../server/dao/tenant/TenantServiceImpl.java | 113 +++--------------- .../usagerecord/ApiUsageStateServiceImpl.java | 2 +- 18 files changed, 63 insertions(+), 160 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index a324f279fb..c9054c30fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -97,12 +97,12 @@ public class DefaultHousekeeperService implements HousekeeperService { for (TbProtoQueueMsg msg : msgs) { log.trace("Processing task: {}", msg); try { - processTask(msg); + processTask(msg.getValue()); } catch (InterruptedException e) { return; } catch (Throwable e) { log.error("Unexpected error during message processing [{}]", msg, e); - reprocessingService.submitForReprocessing(msg, e); + reprocessingService.submitForReprocessing(msg.getValue(), e); } } consumer.commit(); @@ -122,8 +122,7 @@ public class DefaultHousekeeperService implements HousekeeperService { } @SuppressWarnings("unchecked") - protected void processTask(TbProtoQueueMsg queueMsg) throws Exception { - ToHousekeeperServiceMsg msg = queueMsg.getValue(); + protected void processTask(ToHousekeeperServiceMsg msg) throws Exception { HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(task.getTaskType()); if (taskProcessor == null) { @@ -131,8 +130,9 @@ public class DefaultHousekeeperService implements HousekeeperService { } if (log.isDebugEnabled()) { - log.debug("[{}][{}][{}] {} task {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", task.getTaskType()); + log.debug("[{}] {} task {}", task.getTenantId(), + msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", + msg.getTask().getValue()); } try { Future future = executor.submit(() -> { @@ -140,7 +140,7 @@ public class DefaultHousekeeperService implements HousekeeperService { return null; }); future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); - statsService.reportProcessed(task, msg); + statsService.reportProcessed(task.getTaskType(), msg); } catch (InterruptedException e) { throw e; } catch (Throwable e) { @@ -153,8 +153,8 @@ public class DefaultHousekeeperService implements HousekeeperService { log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType(), msg.getTask().getAttempt(), task, error); - reprocessingService.submitForReprocessing(queueMsg, error); - statsService.reportFailure(task, msg); + reprocessingService.submitForReprocessing(msg, error); + statsService.reportFailure(task.getTaskType(), msg); } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index de88b2dcea..4d361cfba4 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -38,6 +38,7 @@ import javax.annotation.PreDestroy; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -107,12 +108,12 @@ public class HousekeeperReprocessingService { for (TbProtoQueueMsg msg : msgs) { log.trace("Reprocessing task: {}", msg); try { - housekeeperService.processTask(msg); + housekeeperService.processTask(msg.getValue()); } catch (InterruptedException e) { return; } catch (Throwable e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); - submitForReprocessing(msg, e); + submitForReprocessing(msg.getValue(), e); // fixme: msgs are duplicated } } @@ -134,8 +135,7 @@ public class HousekeeperReprocessingService { } // todo: dead letter queue if attempts count exceeds the configured maximum - public void submitForReprocessing(TbProtoQueueMsg queueMsg, Throwable error) { - ToHousekeeperServiceMsg msg = queueMsg.getValue(); + public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { HousekeeperTaskProto task = msg.getTask(); int attempt = task.getAttempt() + 1; @@ -152,7 +152,7 @@ public class HousekeeperReprocessingService { log.trace("Submitting for reprocessing: {}", msg); var producer = producerProvider.getHousekeeperReprocessingMsgProducer(); TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); // reprocessing topic has single partition, so we don't care about the msg key } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 4fcc13837d..ab4f699ed1 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -67,8 +67,8 @@ public class HousekeeperStatsService { } } - public void reportProcessed(HousekeeperTask task, ToHousekeeperServiceMsg msg) { - HousekeeperStats stats = this.stats.get(task.getTaskType()); + public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getProcessedCounter().increment(); } else { @@ -76,8 +76,8 @@ public class HousekeeperStatsService { } } - public void reportFailure(HousekeeperTask task, ToHousekeeperServiceMsg msg) { - HousekeeperStats stats = this.stats.get(task.getTaskType()); + public void reportFailure(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getFailedProcessingCounter().increment(); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java index 4c92e9b92b..fc573e0bc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java @@ -47,7 +47,7 @@ public class RuleEngineComponentLifecycleEventTriggerProcessor implements Notifi return false; } } - if (!partitionService.resolve(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId()).isMyPartition()) { + if (!partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId())) { return false; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index 6316ab1281..fb4fe1011e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -58,8 +58,7 @@ public enum EntityType { NOTIFICATION_TEMPLATE (30), NOTIFICATION_REQUEST (31), NOTIFICATION (32), - NOTIFICATION_RULE (33), - ADMIN_SETTINGS(34); + NOTIFICATION_RULE (33); @Getter private final int protoNumber; // Corresponds to EntityTypeProto diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index ceebe8f7b2..c9806a03c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -47,7 +46,6 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -91,10 +89,6 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; - @Lazy - @Autowired - private QueueService queueService; - @Autowired private ImageService imageService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 1097ec5ece..5bdcd341b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -37,13 +37,11 @@ public class CleanUpService { private final HousekeeperService housekeeperService; private final RelationService relationService; - @TransactionalEventListener(fallbackExecution = true) // todo: consider moving this to HousekeeperService + @TransactionalEventListener(fallbackExecution = true) public void handleEntityDeletionEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); - log.trace("[{}] DeleteEntityEvent handler: {}", tenantId, event); - - log.info("[{}][{}][{}] Handling DeleteEntityEvent", tenantId, entityId.getEntityType(), entityId.getId()); + log.trace("[{}] Handling entity deletion event: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); @@ -60,7 +58,7 @@ public class CleanUpService { } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { - UUID tasksKey = UUID.randomUUID(); // so that all tasks are processed synchronously from one partition + UUID tasksKey = UUID.randomUUID(); // so that all tasks are pushed to single partition to be processed synchronously for (EntityType entityType : entityTypes) { housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java index aea09c093c..8761229e89 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.User; @Data @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { private String userTitle; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java index 7846feb229..e20f3ffe0c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java @@ -15,13 +15,16 @@ */ package org.thingsboard.server.dao.housekeeper.data; +import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @Data @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private EntityType entityType; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java index 19a04fd867..7543c247e0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java @@ -42,6 +42,7 @@ public class AssetProfileDataValidator extends DataValidator { @Lazy private AssetProfileService assetProfileService; @Autowired + @Lazy private TenantService tenantService; @Lazy @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java index 178905f7f6..829da4d5c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; @@ -28,6 +29,7 @@ import org.thingsboard.server.dao.tenant.TenantService; public class DashboardDataValidator extends DataValidator { @Autowired + @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index e2047dc448..a8f1bdd972 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -39,6 +40,7 @@ public class DeviceDataValidator extends AbstractHasOtaPackageValidator private DeviceDao deviceDao; @Autowired + @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 23f3a6bbdb..a06fd5edaa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -86,6 +86,7 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { private WidgetTypeDao widgetTypeDao; @Autowired + @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java index 222aaa3eeb..2f758b1b9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java @@ -53,6 +53,7 @@ public class RuleChainDataValidator extends DataValidator { private RuleChainService ruleChainService; @Autowired + @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 1100d40d22..095e51082a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -21,21 +21,15 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.AdminSettingsId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; -import java.util.Optional; - @Service @Slf4j -public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDaoService { - +public class AdminSettingsServiceImpl implements AdminSettingsService { + @Autowired private AdminSettingsDao adminSettingsDao; @@ -46,7 +40,7 @@ public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDao public AdminSettings findAdminSettingsById(TenantId tenantId, AdminSettingsId adminSettingsId) { log.trace("Executing findAdminSettingsById [{}]", adminSettingsId); Validator.validateId(adminSettingsId, "Incorrect adminSettingsId " + adminSettingsId); - return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); + return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); } @Override @@ -65,15 +59,15 @@ public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDao public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if (adminSettings.getKey().equals("mail")){ + if (adminSettings.getKey().equals("mail")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { JsonNode newJsonValue = adminSettings.getJsonValue(); JsonNode oldJsonValue = mailSettings.getJsonValue(); - if (!newJsonValue.has("password") && oldJsonValue.has("password")){ - ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); + if (!newJsonValue.has("password") && oldJsonValue.has("password")) { + ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); } - if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")){ + if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")) { ((ObjectNode) newJsonValue).put("refreshToken", oldJsonValue.get("refreshToken").asText()); } dropTokenIfProviderInfoChanged(newJsonValue, oldJsonValue); @@ -98,12 +92,12 @@ public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDao } private void dropTokenIfProviderInfoChanged(JsonNode newJsonValue, JsonNode oldJsonValue) { - if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()){ + if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()) { if (!newJsonValue.get("providerId").equals(oldJsonValue.get("providerId")) || !newJsonValue.get("clientId").equals(oldJsonValue.get("clientId")) || !newJsonValue.get("clientSecret").equals(oldJsonValue.get("clientSecret")) || !newJsonValue.get("redirectUri").equals(oldJsonValue.get("redirectUri")) || - (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))){ + (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))) { ((ObjectNode) newJsonValue).put("tokenGenerated", false); ((ObjectNode) newJsonValue).remove("refreshToken"); ((ObjectNode) newJsonValue).remove("refreshTokenExpires"); @@ -111,19 +105,4 @@ public class AdminSettingsServiceImpl implements AdminSettingsService, EntityDao } } - @Override - public void deleteByTenantId(TenantId tenantId) { - deleteAdminSettingsByTenantId(tenantId); - } - - @Override - public Optional> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.empty(); - } - - @Override - public EntityType getEntityType() { - return EntityType.ADMIN_SETTINGS; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index b82cfa8613..6ab9361b19 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -16,9 +16,8 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -34,32 +33,17 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.asset.AssetProfileService; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.customer.CustomerService; -import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; -import org.thingsboard.server.dao.notification.NotificationRequestService; -import org.thingsboard.server.dao.notification.NotificationRuleService; import org.thingsboard.server.dao.notification.NotificationSettingsService; -import org.thingsboard.server.dao.notification.NotificationTargetService; -import org.thingsboard.server.dao.notification.NotificationTemplateService; -import org.thingsboard.server.dao.ota.OtaPackageService; -import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.resource.ResourceService; -import org.thingsboard.server.dao.rpc.RpcService; -import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.dao.widget.WidgetTypeService; -import org.thingsboard.server.dao.widget.WidgetsBundleService; import java.util.List; import java.util.Optional; @@ -68,89 +52,22 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service("TenantDaoService") @Slf4j +@RequiredArgsConstructor public class TenantServiceImpl extends AbstractCachedEntityService implements TenantService { private static final String DEFAULT_TENANT_REGION = "Global"; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - @Autowired - private TenantDao tenantDao; - - @Autowired - private TenantProfileService tenantProfileService; - - @Autowired - @Lazy - private UserService userService; - - @Autowired - private CustomerService customerService; - - @Autowired - private AssetService assetService; - - @Autowired - private AssetProfileService assetProfileService; - - @Autowired - private DeviceService deviceService; - - @Autowired - private DeviceProfileService deviceProfileService; - - @Lazy - @Autowired - private ApiUsageStateService apiUsageStateService; - - @Autowired - private WidgetsBundleService widgetsBundleService; - - @Autowired - private WidgetTypeService widgetTypeService; - - @Autowired - private DashboardService dashboardService; - - @Autowired - private RuleChainService ruleChainService; - - @Autowired - private ResourceService resourceService; - - @Autowired - @Lazy - private OtaPackageService otaPackageService; - - @Autowired - private RpcService rpcService; - - @Autowired - private DataValidator tenantValidator; - - @Lazy - @Autowired - private QueueService queueService; - - @Autowired - private AdminSettingsService adminSettingsService; - - @Autowired - private NotificationSettingsService notificationSettingsService; - - @Autowired - private NotificationRequestService notificationRequestService; - - @Autowired - private NotificationRuleService notificationRuleService; - - @Autowired - private NotificationTemplateService notificationTemplateService; - - @Autowired - private NotificationTargetService notificationTargetService; - - @Autowired - protected TbTransactionalCache existsTenantCache; + private final TenantDao tenantDao; + private final TenantProfileService tenantProfileService; + private final UserService userService; + private final AssetProfileService assetProfileService; + private final DeviceProfileService deviceProfileService; + private final ApiUsageStateService apiUsageStateService; + private final AdminSettingsService adminSettingsService; + private final NotificationSettingsService notificationSettingsService; + private final DataValidator tenantValidator; + private final TbTransactionalCache existsTenantCache; @TransactionalEventListener(classes = TenantEvictEvent.class) @Override @@ -218,16 +135,18 @@ public class TenantServiceImpl extends AbstractCachedEntityService apiUsageStateValidator; public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, - TenantService tenantService, @Lazy TimeseriesService tsService, + @Lazy TenantService tenantService, @Lazy TimeseriesService tsService, DataValidator apiUsageStateValidator) { this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; From b20b6006e9b79e06267e077e1c36cb58e2e9c00a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 1 Mar 2024 14:14:25 +0200 Subject: [PATCH 13/72] Notifications to sysadmin on housekeeper task processing failure --- .../install/ThingsboardInstallService.java | 7 +- .../DefaultHousekeeperService.java | 19 ++++-- .../AlarmsUnassignTaskProcessor.java | 4 +- .../AttributesDeletionTaskProcessor.java | 4 +- .../EntitiesDeletionTaskProcessor.java | 4 +- .../EntityAlarmsDeletionTaskProcessor.java | 4 +- .../EventsDeletionTaskProcessor.java | 4 +- .../processor/HousekeeperTaskProcessor.java | 4 +- .../TelemetryDeletionTaskProcessor.java | 4 +- .../stats/HousekeeperStatsService.java | 3 +- .../DefaultSystemDataLoaderService.java | 37 ++++++----- .../install/SystemDataLoaderService.java | 2 +- ...TaskProcessingFailureTriggerProcessor.java | 53 +++++++++++++++ .../NotificationTemplateService.java | 2 + .../AlarmsUnassignHousekeeperTask.java | 2 +- .../EntitiesDeletionHousekeeperTask.java | 9 ++- .../data/housekeeper}/HousekeeperTask.java | 8 ++- .../data/housekeeper/HousekeeperTaskType.java | 34 ++++++++++ .../data/notification/NotificationType.java | 3 +- ...TaskProcessingFailureNotificationInfo.java | 61 +++++++++++++++++ .../trigger/TaskProcessingFailureTrigger.java | 65 +++++++++++++++++++ .../config/NotificationRuleTriggerConfig.java | 1 + .../config/NotificationRuleTriggerType.java | 3 +- ...gFailureNotificationRuleTriggerConfig.java | 21 +++--- .../dao/housekeeper/CleanUpService.java | 28 +++++--- .../dao/housekeeper/HousekeeperService.java | 6 +- .../DefaultNotificationSettingsService.java | 24 +++---- .../DefaultNotificationTemplateService.java | 5 ++ .../notification/DefaultNotifications.java | 14 ++++ .../notification/NotificationTemplateDao.java | 2 + .../JpaNotificationTemplateDao.java | 5 ++ .../NotificationTemplateRepository.java | 5 ++ .../rule-notification-dialog.component.html | 13 ++++ .../rule-notification-dialog.component.ts | 13 +++- .../template-notification-dialog.component.ts | 3 +- .../app/shared/models/notification.models.ts | 18 +++-- .../notification/task_processing_failure.md | 46 +++++++++++++ .../assets/locale/locale.constant-en_US.json | 5 +- 38 files changed, 458 insertions(+), 87 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/AlarmsUnassignHousekeeperTask.java (95%) rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/EntitiesDeletionHousekeeperTask.java (83%) rename {dao/src/main/java/org/thingsboard/server/dao/housekeeper/data => common/data/src/main/java/org/thingsboard/server/common/data/housekeeper}/HousekeeperTask.java (91%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java rename dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java => common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java (61%) create mode 100644 ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 44d81902a8..a79ed50eb7 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -110,7 +110,7 @@ public class ThingsboardInstallService { log.info("Upgrading ThingsBoard from version 3.5.1 to 3.6.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.5.1"); dataUpdateService.updateData("3.5.1"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); @@ -126,7 +126,10 @@ public class ThingsboardInstallService { case "3.6.2": log.info("Upgrading ThingsBoard from version 3.6.2 to 3.6.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); + case "3.6.3": + log.info("Upgrading ThingsBoard from version 3.6.3 to 3.6.4 ..."); + systemDataLoaderService.updateDefaultNotificationConfigs(false); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index c9054c30fa..56123af64d 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -21,10 +21,12 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.housekeeper.HousekeeperService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; @@ -60,6 +62,7 @@ public class DefaultHousekeeperService implements HousekeeperService { private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; private final HousekeeperStatsService statsService; + private final NotificationRuleProcessor notificationRuleProcessor; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @@ -75,11 +78,13 @@ public class DefaultHousekeeperService implements HousekeeperService { TbCoreQueueFactory queueFactory, TbQueueProducerProvider producerProvider, HousekeeperStatsService statsService, + NotificationRuleProcessor notificationRuleProcessor, @Lazy List> taskProcessors) { this.consumer = queueFactory.createHousekeeperMsgConsumer(); this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; this.statsService = statsService; + this.notificationRuleProcessor = notificationRuleProcessor; this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); } @@ -155,14 +160,14 @@ public class DefaultHousekeeperService implements HousekeeperService { task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(msg, error); statsService.reportFailure(task.getTaskType(), msg); + notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() + .task(task) + .error(error) + .attempt(msg.getTask().getAttempt()) + .build()); } } - @Override - public void submitTask(HousekeeperTask task) { - submitTask(task.getEntityId().getId(), task); - } - @Override public void submitTask(UUID key, HousekeeperTask task) { log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 68fd05deec..9a8517903f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -20,8 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; -import org.thingsboard.server.dao.housekeeper.data.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask; import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index 0467e50aeb..c8ff5306a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -19,8 +19,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index da7af3c42b..8028a76199 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -19,8 +19,8 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; -import org.thingsboard.server.dao.housekeeper.data.EntitiesDeletionHousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java index a91f2262f9..974a95eb08 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java @@ -18,8 +18,8 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 481f9f1bbf..551c8ce7e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -18,8 +18,8 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index 7daa99540e..79e87fcc91 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.housekeeper.processor; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; public interface HousekeeperTaskProcessor { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index e7da740261..c7f37ed091 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -20,8 +20,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index ab4f699ed1..3f6e88a299 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -23,8 +23,7 @@ import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import java.util.ArrayList; diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index b20a548d90..45ae1280b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -548,7 +548,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { } else { ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, DataConstants.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) - , System.currentTimeMillis()))); + , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -692,23 +692,26 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override @SneakyThrows - public void updateDefaultNotificationConfigs() { - PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); - ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); - log.info("Updating default edge failure notification configs for all tenants"); - AtomicInteger count = new AtomicInteger(); - for (TenantId tenantId : tenants) { - executor.submit(() -> { - notificationSettingsService.updateDefaultNotificationConfigs(tenantId); - int n = count.incrementAndGet(); - if (n % 500 == 0) { - log.info("{} tenants processed", n); - } - }); - } - executor.shutdown(); - executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + public void updateDefaultNotificationConfigs(boolean updateTenants) { + log.info("Updating notification configs..."); notificationSettingsService.updateDefaultNotificationConfigs(TenantId.SYS_TENANT_ID); + + if (updateTenants) { + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); + AtomicInteger count = new AtomicInteger(); + for (TenantId tenantId : tenants) { + executor.submit(() -> { + notificationSettingsService.updateDefaultNotificationConfigs(tenantId); + int n = count.incrementAndGet(); + if (n % 500 == 0) { + log.info("{} tenants processed", n); + } + }); + } + executor.shutdown(); + executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index eeac1b6aa4..7c05ff620f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -35,6 +35,6 @@ public interface SystemDataLoaderService { void createDefaultNotificationConfigs(); - void updateDefaultNotificationConfigs(); + void updateDefaultNotificationConfigs(boolean updateTenants); } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java new file mode 100644 index 0000000000..7754690215 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.notification.rule.trigger; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.notification.info.TaskProcessingFailureNotificationInfo; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; + +@Service +public class TaskProcessingFailureTriggerProcessor implements NotificationRuleTriggerProcessor { + + @Override + public boolean matchesFilter(TaskProcessingFailureTrigger trigger, TaskProcessingFailureNotificationRuleTriggerConfig triggerConfig) { + return true; + } + + @Override + public TaskProcessingFailureNotificationInfo constructNotificationInfo(TaskProcessingFailureTrigger trigger) { + HousekeeperTask task = trigger.getTask(); + return TaskProcessingFailureNotificationInfo.builder() + .tenantId(task.getTenantId()) + .entityId(task.getEntityId()) + .taskType(task.getTaskType()) + .taskDescription(task.getDescription()) + .error(StringUtils.truncate(ExceptionUtils.getStackTrace(trigger.getError()), 1024)) + .attempt(trigger.getAttempt()) + .build(); + } + + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java index 1f4019063b..664f976904 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java @@ -32,6 +32,8 @@ public interface NotificationTemplateService { PageData findNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id); void deleteNotificationTemplatesByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java similarity index 95% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 8761229e89..af4e746987 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; import lombok.AccessLevel; import lombok.Data; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java similarity index 83% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index e20f3ffe0c..515703991c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; @@ -34,4 +35,10 @@ public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { this.entityType = entityType; } + @JsonIgnore + @Override + public String getDescription() { + return entityType.getNormalName().toLowerCase() + "s deletion"; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java similarity index 91% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java rename to common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 6b63d43d43..44598d511c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.housekeeper; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -76,4 +77,9 @@ public class HousekeeperTask implements Serializable { return new EntitiesDeletionHousekeeperTask(tenantId, entityType); } + @JsonIgnore + public String getDescription() { + return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java new file mode 100644 index 0000000000..4f30f94a45 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Getter +public enum HousekeeperTaskType { + + DELETE_ENTITIES("entities deletion"), + DELETE_ATTRIBUTES("attributes deletion"), + DELETE_TELEMETRY("telemetry deletion"), + DELETE_EVENTS("events deletion"), + UNASSIGN_ALARMS("alarms unassigning"), + DELETE_ENTITY_ALARMS("entity alarms deletion"); + + private final String description; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java index 5a8fd87933..ec66a3d53a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java @@ -30,6 +30,7 @@ public enum NotificationType { RULE_NODE, RATE_LIMITS, EDGE_CONNECTION, - EDGE_COMMUNICATION_FAILURE + EDGE_COMMUNICATION_FAILURE, + TASK_PROCESSING_FAILURE } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java new file mode 100644 index 0000000000..cbee861c04 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.info; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +import static org.thingsboard.server.common.data.util.CollectionsUtil.mapOf; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class TaskProcessingFailureNotificationInfo implements RuleOriginatedNotificationInfo { + + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private String taskDescription; + private String error; + private int attempt; + + @Override + public Map getTemplateData() { + return mapOf( + "tenantId", tenantId.toString(), + "entityType", entityId.getEntityType().getNormalName(), + "entityId", entityId.getId().toString(), + "taskType", taskType.getDescription(), + "taskDescription", taskDescription, + "error", error, + "attempt", String.valueOf(attempt) + ); + } + + @Override + public TenantId getAffectedTenantId() { + return tenantId; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java new file mode 100644 index 0000000000..9ccfac6aff --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.rule.trigger; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; + +import java.util.concurrent.TimeUnit; + +@Data +@Builder +public class TaskProcessingFailureTrigger implements NotificationRuleTrigger { + + private final HousekeeperTask task; + private final int attempt; + private final Throwable error; + + @Override + public NotificationRuleTriggerType getType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + + @Override + public TenantId getTenantId() { + return task.getTenantId(); + } + + @Override + public EntityId getOriginatorEntityId() { + return task.getEntityId(); + } + + @Override + public boolean deduplicate() { + return true; + } + + @Override + public String getDeduplicationKey() { + return String.join(":", NotificationRuleTrigger.super.getDeduplicationKey(), task.getTaskType().toString()); + } + + @Override + public long getDefaultDeduplicationDuration() { + return TimeUnit.MINUTES.toMillis(30); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java index 15a5e59255..1621ec8120 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java @@ -38,6 +38,7 @@ import java.io.Serializable; @Type(value = RateLimitsNotificationRuleTriggerConfig.class, name = "RATE_LIMITS"), @Type(value = EdgeConnectionNotificationRuleTriggerConfig.class, name = "EDGE_CONNECTION"), @Type(value = EdgeCommunicationFailureNotificationRuleTriggerConfig.class, name = "EDGE_COMMUNICATION_FAILURE"), + @Type(value = TaskProcessingFailureNotificationRuleTriggerConfig.class, name = "TASK_PROCESSING_FAILURE") }) public interface NotificationRuleTriggerConfig extends Serializable { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java index 8469fac752..8846ae284d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java @@ -31,7 +31,8 @@ public enum NotificationRuleTriggerType { NEW_PLATFORM_VERSION(false), ENTITIES_LIMIT(false), API_USAGE_LIMIT(false), - RATE_LIMITS(false); + RATE_LIMITS(false), + TASK_PROCESSING_FAILURE(false); private final boolean tenantLevel; diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java similarity index 61% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java index 51fb9ab926..73f0517238 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/data/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java @@ -13,13 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper.data; +package org.thingsboard.server.common.data.notification.rule.trigger.config; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class TaskProcessingFailureNotificationRuleTriggerConfig implements NotificationRuleTriggerConfig { + + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } -public enum HousekeeperTaskType { - DELETE_ENTITIES, - DELETE_ATTRIBUTES, - DELETE_TELEMETRY, // maybe divide into latest and ts kv history? - DELETE_EVENTS, - UNASSIGN_ALARMS, - DELETE_ENTITY_ALARMS } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 5bdcd341b8..b81b6dc5e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -21,12 +21,13 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; import org.thingsboard.server.dao.relation.RelationService; +import java.util.Optional; import java.util.UUID; @Component @@ -34,7 +35,7 @@ import java.util.UUID; @Slf4j public class CleanUpService { - private final HousekeeperService housekeeperService; + private final Optional housekeeperService; private final RelationService relationService; @TransactionalEventListener(fallbackExecution = true) @@ -44,24 +45,31 @@ public class CleanUpService { log.trace("[{}] Handling entity deletion event: {}", tenantId, event); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { - housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + housekeeperService.ifPresent(housekeeperService -> { + housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + }); } } public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + housekeeperService.ifPresent(housekeeperService -> { + housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); + }); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { UUID tasksKey = UUID.randomUUID(); // so that all tasks are pushed to single partition to be processed synchronously - for (EntityType entityType : entityTypes) { - housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); - } + // todo: just use tenantId as key in the impl + housekeeperService.ifPresent(housekeeperService -> { + for (EntityType entityType : entityTypes) { + housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); + } + }); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java index 5e55318953..1bbbb25df1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -15,13 +15,15 @@ */ package org.thingsboard.server.dao.housekeeper; -import org.thingsboard.server.dao.housekeeper.data.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import java.util.UUID; public interface HousekeeperService { - void submitTask(HousekeeperTask task); + default void submitTask(HousekeeperTask task) { + submitTask(task.getEntityId().getId(), task); + } // tasks with the same key will be pushed to the same partition and thus processed synchronously void submitTask(UUID key, HousekeeperTask task); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java index 1bdc835b29..a37d37e849 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java @@ -173,17 +173,15 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForTenant, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.newPlatformVersion, sysAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, tenantAdmins.getId()); return; } @@ -206,19 +204,19 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS @Override public void updateDefaultNotificationConfigs(TenantId tenantId) { if (tenantId.isSysTenantId()) { - if (notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, - List.of(NotificationType.RATE_LIMITS), new PageLink(1)).getTotalElements() > 0) { - return; - } - NotificationTarget sysAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.SYSTEM_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "System administrators", new SystemAdministratorsFilter(), "All system administrators")); NotificationTarget affectedTenantAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.AFFECTED_TENANT_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "Affected tenant's administrators", new AffectedTenantAdministratorsFilter(), "")); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + if (!isNotificationConfigured(tenantId, NotificationType.RATE_LIMITS)) { + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + } + if (!isNotificationConfigured(tenantId, NotificationType.TASK_PROCESSING_FAILURE)) { + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, sysAdmins.getId()); + } } else { var requiredNotificationTypes = List.of(NotificationType.EDGE_CONNECTION, NotificationType.EDGE_COMMUNICATION_FAILURE); var existingNotificationTypes = notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes( @@ -252,6 +250,10 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS } } + private boolean isNotificationConfigured(TenantId tenantId, NotificationType... notificationTypes) { + return notificationTemplateService.countNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, List.of(notificationTypes)) > 0; + } + private NotificationTarget createTarget(TenantId tenantId, String name, UsersFilter filter, String description) { NotificationTarget target = new NotificationTarget(); target.setTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index 6cf8239f47..bec6c3d61c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -69,6 +69,11 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im return notificationTemplateDao.findByTenantIdAndNotificationTypesAndPageLink(tenantId, notificationTypes, pageLink); } + @Override + public int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes) { + return notificationTemplateDao.countByTenantIdAndNotificationTypes(tenantId, notificationTypes); + } + @Override public void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id) { if (notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index fd17618bd1..4f617c2b9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; import org.thingsboard.server.common.data.notification.rule.trigger.config.RateLimitsNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.rule.trigger.config.RuleEngineComponentLifecycleEventNotificationRuleTriggerConfig; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.common.data.notification.template.NotificationTemplateConfig; import org.thingsboard.server.common.data.notification.template.WebDeliveryMethodNotificationTemplate; @@ -358,6 +359,19 @@ public class DefaultNotifications { .build()) .build(); + public static final DefaultNotification taskProcessingFailure = DefaultNotification.builder() + .name("Task processing failure notification") + .type(NotificationType.TASK_PROCESSING_FAILURE) + .subject("Failed to process ${taskType}") + .text("Failed to process ${taskDescription} for tenant ${tenantId}: ${error}") + .icon("warning").color(YELLOW_COLOR) + .rule(DefaultRule.builder() + .name("Task processing failure") + .triggerConfig(TaskProcessingFailureNotificationRuleTriggerConfig.builder().build()) + .description("Send notification to system admins when task processing fails") + .build()) + .build(); + public static final DefaultNotification jwtSigningKeyIssue = DefaultNotification.builder() .name("JWT Signing Key issue notification") .type(NotificationType.GENERAL) diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java index d9a80ce3f5..136bc4d472 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java @@ -30,6 +30,8 @@ public interface NotificationTemplateDao extends Dao, Expo PageData findByTenantIdAndNotificationTypesAndPageLink(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void removeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java index 8ee6c3df3a..8950ab1e35 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java @@ -53,6 +53,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao notificationTypes) { + return notificationTemplateRepository.countByTenantIdAndNotificationTypes(tenantId.getId(), notificationTypes); + } + @Override public void removeByTenantId(TenantId tenantId) { notificationTemplateRepository.deleteByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java index b3e416a491..bb45c10d06 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java @@ -42,6 +42,11 @@ public interface NotificationTemplateRepository extends JpaRepository notificationTypes); + @Transactional @Modifying @Query("DELETE FROM NotificationTemplateEntity t WHERE t.tenantId = :tenantId") diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html index 5b477d0345..f5a2ddba17 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html @@ -581,6 +581,19 @@ + + + {{ 'notification.task-processing-failure-trigger-settings' | translate }} +
+
+ + notification.description + + +
+
+ diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts index 89488b7517..bc6935b4e7 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts @@ -101,6 +101,7 @@ export class RuleNotificationDialogComponent extends rateLimitsTemplateForm: FormGroup; edgeCommunicationFailureTemplateForm: FormGroup; edgeConnectionTemplateForm: FormGroup; + taskProcessingFailureTemplateForm: FormGroup; triggerType = TriggerType; triggerTypes: TriggerType[]; @@ -337,6 +338,12 @@ export class RuleNotificationDialogComponent extends }) }); + this.taskProcessingFailureTemplateForm = this.fb.group({ + triggerConfig: this.fb.group({ + taskTypes: [] + }) + }); + this.triggerTypeFormsMap = new Map([ [TriggerType.ALARM, this.alarmTemplateForm], [TriggerType.ALARM_COMMENT, this.alarmCommentTemplateForm], @@ -349,7 +356,8 @@ export class RuleNotificationDialogComponent extends [TriggerType.NEW_PLATFORM_VERSION, this.newPlatformVersionTemplateForm], [TriggerType.RATE_LIMITS, this.rateLimitsTemplateForm], [TriggerType.EDGE_COMMUNICATION_FAILURE, this.edgeCommunicationFailureTemplateForm], - [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm] + [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm], + [TriggerType.TASK_PROCESSING_FAILURE, this.taskProcessingFailureTemplateForm] ]); if (data.isAdd || data.isCopy) { @@ -488,7 +496,8 @@ export class RuleNotificationDialogComponent extends TriggerType.ENTITIES_LIMIT, TriggerType.API_USAGE_LIMIT, TriggerType.NEW_PLATFORM_VERSION, - TriggerType.RATE_LIMITS + TriggerType.RATE_LIMITS, + TriggerType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts index c0ae4f1abb..ccbcb6bf22 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts @@ -179,7 +179,8 @@ export class TemplateNotificationDialogComponent NotificationType.ENTITIES_LIMIT, NotificationType.API_USAGE_LIMIT, NotificationType.NEW_PLATFORM_VERSION, - NotificationType.RATE_LIMITS + NotificationType.RATE_LIMITS, + NotificationType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/shared/models/notification.models.ts b/ui-ngx/src/app/shared/models/notification.models.ts index 2cc7d8547b..0c5e1c5a90 100644 --- a/ui-ngx/src/app/shared/models/notification.models.ts +++ b/ui-ngx/src/app/shared/models/notification.models.ts @@ -523,7 +523,8 @@ export enum NotificationType { RULE_NODE = 'RULE_NODE', RATE_LIMITS = 'RATE_LIMITS', EDGE_CONNECTION = 'EDGE_CONNECTION', - EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE' + EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE', + TASK_PROCESSING_FAILURE = 'TASK_PROCESSING_FAILURE' } export const NotificationTypeIcons = new Map([ @@ -535,6 +536,7 @@ export const NotificationTypeIcons = new Map([ [NotificationType.RULE_ENGINE_COMPONENT_LIFECYCLE_EVENT, 'settings_ethernet'], [NotificationType.ENTITIES_LIMIT, 'data_thresholding'], [NotificationType.API_USAGE_LIMIT, 'insert_chart'], + [NotificationType.TASK_PROCESSING_FAILURE, 'warning'] ]); export const AlarmSeverityNotificationColors = new Map( @@ -646,7 +648,13 @@ export const NotificationTemplateTypeTranslateMap = new Map([ @@ -676,7 +685,8 @@ export const TriggerTypeTranslationMap = new Map([ [TriggerType.NEW_PLATFORM_VERSION, 'notification.trigger.new-platform-version'], [TriggerType.RATE_LIMITS, 'notification.trigger.rate-limits'], [TriggerType.EDGE_CONNECTION, 'notification.trigger.edge-connection'], - [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'] + [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'], + [TriggerType.TASK_PROCESSING_FAILURE, 'notification.trigger.task-processing-failure'] ]); export interface NotificationUserSettings { diff --git a/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md new file mode 100644 index 0000000000..0bcf107298 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md @@ -0,0 +1,46 @@ +#### Task processing failure notification templatization + +
+
+ +Notification subject and message fields support templatization. +The list of available templatization parameters depends on the template type. +See the available types and parameters below: + +Available template parameters: + +* `taskType` - the task type, e.g. 'telemetry deletion'; +* `taskDescription` - the task description, e.g. 'telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325'; +* `error` - the error stacktrace +* `tenantId` - the tenant id; +* `entityType` - the type of the entity to which the task is related; +* `entityId` - the id of the entity to which the task is related; +* `attempt` - the number of attempts processing the task + +Parameter names must be wrapped using `${...}`. For example: `${entityType}`. +You may also modify the value of the parameter with one of the suffixes: + +* `upperCase`, for example - `${entityType:upperCase}` +* `lowerCase`, for example - `${entityType:lowerCase}` +* `capitalize`, for example - `${entityType:capitalize}` + +
+ +##### Examples + +Let's assume that telemetry deletion failed for some device. +The following template: + +```text +Failed to process ${taskType} for ${entityType:lowerCase} ${entityId} +{:copy-code} +``` + +will be transformed to: + +```text +Failed to process telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325 +``` + +
+
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0428087620..f94a2fffd3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3241,6 +3241,7 @@ "api-usage-trigger-settings": "API usage trigger settings", "new-platform-version-trigger-settings": "New platform version trigger settings", "rate-limits-trigger-settings": "Exceeded rate limits trigger settings", + "task-processing-failure-trigger-settings": "Task processing failure trigger settings", "at-least-one-should-be-selected": "At least one should be selected", "basic-settings": "Basic settings", "button-text": "Button text", @@ -3450,7 +3451,8 @@ "new-platform-version": "New platform version", "rate-limits": "Exceeded rate limits", "edge-communication-failure": "Edge communication failure", - "edge-connection": "Edge connection" + "edge-connection": "Edge connection", + "task-processing-failure": "Task processing failure" }, "templates": "Templates", "notification-templates": "Notifications / Templates", @@ -3473,6 +3475,7 @@ "rate-limits": "Exceeded rate limits", "edge-connection": "Edge connection", "edge-communication-failure": "Edge communication failure", + "task-processing-failure": "Task processing failure", "trigger": "Trigger", "trigger-required": "Trigger is required" }, From b82bceceb4142e31167293871a304d2f63cbda89 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 7 Mar 2024 17:01:33 +0200 Subject: [PATCH 14/72] Housekeeper: tests and refactoring --- .../DefaultHousekeeperService.java | 20 +- .../EntitiesDeletionTaskProcessor.java | 7 +- .../EntityAlarmsDeletionTaskProcessor.java | 2 + .../DefaultExportableEntitiesService.java | 3 - .../housekeeper/HousekeeperServiceTest.java | 303 ++++++++++++++++++ .../entity/DefaultEntityServiceRegistry.java | 4 +- .../dao/housekeeper/CleanUpService.java | 22 +- .../dao/housekeeper/HousekeeperService.java | 9 +- .../server/dao/rule/BaseRuleChainService.java | 8 +- 9 files changed, 344 insertions(+), 34 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 56123af64d..63e8bb6a75 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -42,7 +42,6 @@ import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -58,11 +57,12 @@ public class DefaultHousekeeperService implements HousekeeperService { private final Map> taskProcessors; - private final TbQueueConsumer> consumer; - private final TbQueueProducer> producer; private final HousekeeperReprocessingService reprocessingService; private final HousekeeperStatsService statsService; private final NotificationRuleProcessor notificationRuleProcessor; + private final TbQueueConsumer> consumer; + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @@ -80,11 +80,12 @@ public class DefaultHousekeeperService implements HousekeeperService { HousekeeperStatsService statsService, NotificationRuleProcessor notificationRuleProcessor, @Lazy List> taskProcessors) { - this.consumer = queueFactory.createHousekeeperMsgConsumer(); - this.producer = producerProvider.getHousekeeperMsgProducer(); this.reprocessingService = reprocessingService; this.statsService = statsService; this.notificationRuleProcessor = notificationRuleProcessor; + this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); } @@ -169,10 +170,13 @@ public class DefaultHousekeeperService implements HousekeeperService { } @Override - public void submitTask(UUID key, HousekeeperTask task) { + public void submitTask(HousekeeperTask task) { log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); - TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(key, ToHousekeeperServiceMsg.newBuilder() + /* + * using msg key as entity id so that msgs related to certain entity are pushed to same partition, + * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order + * */ + producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() .setTask(HousekeeperTaskProto.newBuilder() .setValue(JacksonUtil.toString(task)) .setTs(task.getTs()) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index 8028a76199..a16a5f09bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -17,10 +17,10 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.entity.EntityDaoService; -import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; @Component @RequiredArgsConstructor @@ -31,9 +31,6 @@ public class EntitiesDeletionTaskProcessor implements HousekeeperTaskProcessor void removeById(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); - if (entityService == null) { - throw new IllegalArgumentException("Unsupported entity type " + entityType); - } entityService.deleteEntity(tenantId, id); } diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java new file mode 100644 index 0000000000..b132cf34fd --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -0,0 +1,303 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbNodeConnectionType; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.HousekeeperService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +@TestPropertySource(properties = { + "transport.http.enabled=true" +}) +public class HousekeeperServiceTest extends AbstractControllerTest { + + @SpyBean + private HousekeeperService housekeeperService; + @Autowired + private EventService eventService; + @Autowired + private TimeseriesService timeseriesService; + @Autowired + private AttributesService attributesService; + @Autowired + private RuleChainService ruleChainService; + @Autowired + private AlarmService alarmService; + + private TenantId tenantId; + + private static final String TELEMETRY_KEY = "telemetry1"; + private static final String ATTRIBUTE_KEY = "_attribute1"; + private static final String KV_VALUE = "ewfewfwef"; + + @Before + public void setUp() throws Exception { + loginTenantAdmin(); + this.tenantId = super.tenantId; + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void whenDeviceIsDeleted_thenCleanUpRelatedData() throws Exception { + Device device = createDevice("test", "test"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(device.getId()); + }); + } + + @Test + public void whenRuleChainIsDeleted_thenCleanUpRelatedData() throws Exception { + RuleChainMetaData ruleChainMetaData = createRuleChain(); + RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); + RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId(); + RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId(); + createRelatedData(ruleChainId); + createRelatedData(ruleNode1Id); + createRelatedData(ruleNode2Id); + + doDelete("/api/ruleChain/" + ruleChainId).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(ruleNode1Id); + verifyNoRelatedData(ruleNode2Id); + verifyNoRelatedData(ruleChainId); + }); + } + + @Test + public void whenUserIsDeleted_thenCleanUpRelatedData() throws Exception { + Device device = createDevice("test", "test"); + UserId userId = customerUserId; + createRelatedData(userId); + Alarm alarm = Alarm.builder() + .type("test") + .tenantId(tenantId) + .originator(device.getId()) + .severity(AlarmSeverity.MAJOR) + .build(); + alarm = doPost("/api/alarm", alarm, Alarm.class); + alarm = doPost("/api/alarm/" + alarm.getId() + "/assign/" + userId, "", Alarm.class); + assertThat(alarm.getAssigneeId()).isEqualTo(userId); + assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(100)).getData()).isNotEmpty(); + + doDelete("/api/user/" + userId).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(userId); + }); + assertThat(alarmService.findAlarmById(tenantId, alarm.getId()).getAssigneeId()).isNull(); + } + + @Test + public void whenTenantIsDeleted_thenDeleteAllEntitiesAndCleanUpRelatedData() throws Exception { + loginDifferentTenant(); + tenantId = differentTenantId; + + createRelatedData(tenantId); + + Device device = createDevice("test", "test"); + createRelatedData(device.getId()); + + RuleChainMetaData ruleChainMetaData = createRuleChain(); + RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); + RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId(); + RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId(); + createRelatedData(ruleChainId); + createRelatedData(ruleNode1Id); + createRelatedData(ruleNode2Id); + + UserId userId = savedDifferentTenantUser.getId(); + createRelatedData(userId); + + loginSysAdmin(); + deleteDifferentTenant(); + + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(device.getId()); + verifyNoRelatedData(ruleNode1Id); + verifyNoRelatedData(ruleNode2Id); + verifyNoRelatedData(ruleChainId); + verifyNoRelatedData(userId); + verifyNoRelatedData(tenantId); + }); + } + + private void createRelatedData(EntityId entityId) throws Exception { + createTelemetry(entityId); + for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { + createAttribute(entityId, scope, scope + ATTRIBUTE_KEY); + } + createEvent(entityId); + } + + private void verifyNoRelatedData(EntityId entityId) throws Exception { + List expectedTaskTypes = List.of(HousekeeperTaskType.DELETE_TELEMETRY, HousekeeperTaskType.DELETE_ATTRIBUTES, HousekeeperTaskType.DELETE_EVENTS, HousekeeperTaskType.DELETE_ENTITY_ALARMS); + for (HousekeeperTaskType taskType : expectedTaskTypes) { + verify(housekeeperService).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); + } + + assertThat(getLatestTelemetry(entityId)).isNull(); + assertThat(getTimeseriesHistory(entityId)).isEmpty(); + for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { + assertThat(getAttribute(entityId, scope, scope + ATTRIBUTE_KEY)).isNull(); + } + assertThat(getEvents(entityId)).isEmpty(); + } + + private void createAttribute(EntityId entityId, String scope, String key) throws Exception { + attributesService.save(tenantId, entityId, scope, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, KV_VALUE))).get(); + } + + private void createTelemetry(EntityId entityId) throws Exception { + timeseriesService.save(tenantId, entityId, new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(TELEMETRY_KEY, KV_VALUE))).get(); + } + + private void createEvent(EntityId entityId) { + LifecycleEvent event = LifecycleEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId("test") + .lcEventType("test") + .success(true) + .build(); + eventService.saveAsync(event); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> !getEvents(entityId).isEmpty()); + + } + + private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { + return timeseriesService.findLatest(tenantId, entityId, HousekeeperServiceTest.TELEMETRY_KEY).get().orElse(null); + } + + private List getTimeseriesHistory(EntityId entityId) throws Exception { + return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(HousekeeperServiceTest.TELEMETRY_KEY, 0, System.currentTimeMillis(), 10, "DESC"))).get(); + } + + private AttributeKvEntry getAttribute(EntityId entityId, String scope, String key) throws Exception { + return attributesService.find(tenantId, entityId, scope, key).get().orElse(null); + } + + private List getEvents(EntityId entityId) { + return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100)).getData() + .stream().filter(event -> Optional.ofNullable(event.getBody()).map(body -> body.get("event")) + .map(JsonNode::asText).orElse("").equals("test")) + .collect(Collectors.toList()); + } + + private RuleChainMetaData createRuleChain() { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName("Test"); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + RuleChainId ruleChainId = ruleChain.getId(); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChainId); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(JacksonUtil.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(JacksonUtil.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, TbNodeConnectionType.SUCCESS); + ruleChainService.saveRuleChainMetaData(tenantId, metaData, Function.identity()); + return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index 6e6233b1b2..40064fbbde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.EntityType; import java.util.HashMap; import java.util.Map; +import java.util.Optional; @Service @RequiredArgsConstructor @@ -52,7 +53,8 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { @Override public EntityDaoService getServiceByEntityType(EntityType entityType) { - return entityDaoServicesMap.get(entityType); + return Optional.ofNullable(entityDaoServicesMap.get(entityType)) + .orElseThrow(() -> new IllegalArgumentException("Unsupported entity type " + entityType)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index b81b6dc5e6..13a1e1cb09 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.housekeeper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; @@ -24,11 +26,12 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.relation.RelationService; import java.util.Optional; -import java.util.UUID; @Component @RequiredArgsConstructor @@ -37,12 +40,15 @@ public class CleanUpService { private final Optional housekeeperService; private final RelationService relationService; + @Autowired + @Lazy + private EntityServiceRegistry entityServiceRegistry; @TransactionalEventListener(fallbackExecution = true) public void handleEntityDeletionEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); - log.trace("[{}] Handling entity deletion event: {}", tenantId, event); + log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityId.getEntityType(), entityId.getId()); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { housekeeperService.ifPresent(housekeeperService -> { @@ -52,6 +58,7 @@ public class CleanUpService { } public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + log.trace("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); housekeeperService.ifPresent(housekeeperService -> { @@ -63,11 +70,14 @@ public class CleanUpService { } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { - UUID tasksKey = UUID.randomUUID(); // so that all tasks are pushed to single partition to be processed synchronously - // todo: just use tenantId as key in the impl - housekeeperService.ifPresent(housekeeperService -> { + housekeeperService.ifPresentOrElse(housekeeperService -> { + for (EntityType entityType : entityTypes) { + housekeeperService.submitTask(HousekeeperTask.deleteEntities(tenantId, entityType)); + } + }, () -> { for (EntityType entityType : entityTypes) { - housekeeperService.submitTask(tasksKey, HousekeeperTask.deleteEntities(tenantId, entityType)); + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); + entityService.deleteByTenantId(tenantId); } }); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java index 1bbbb25df1..e3c0800f69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java @@ -17,15 +17,8 @@ package org.thingsboard.server.dao.housekeeper; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; -import java.util.UUID; - public interface HousekeeperService { - default void submitTask(HousekeeperTask task) { - submitTask(task.getEntityId().getId(), task); - } - - // tasks with the same key will be pushed to the same partition and thus processed synchronously - void submitTask(UUID key, HousekeeperTask task); + void submitTask(HousekeeperTask task); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 92d4ea1baf..18f3d58ba7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -438,12 +438,14 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC checkRuleNodesAndDelete(tenantId, ruleChainId); } + @Transactional @Override public void deleteRuleChainsByTenantId(TenantId tenantId) { Validator.validateId(tenantId, "Incorrect tenant id for delete rule chains request."); tenantRuleChainsRemover.removeEntities(tenantId, tenantId); } + @Transactional @Override public void deleteByTenantId(TenantId tenantId) { deleteRuleChainsByTenantId(tenantId); @@ -776,10 +778,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); - for (var node : ruleNodes) { - cleanUpService.cleanUpRelatedData(tenantId, node.getId()); - } ruleNodeDao.deleteByIdIn(ruleNodeIds); + for (var nodeId : ruleNodeIds) { + cleanUpService.cleanUpRelatedData(tenantId, nodeId); + } } @Override From 0d3c3d6b25c24235bbfcd106ce24d354a82ebca7 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 13 Mar 2024 14:09:05 +0200 Subject: [PATCH 15/72] Housekeeper: tests and refactoring --- .../DefaultHousekeeperService.java | 22 ++++-- .../HousekeeperReprocessingService.java | 34 +++++++--- .../src/main/resources/thingsboard.yml | 9 ++- .../housekeeper/HousekeeperServiceTest.java | 68 +++++++++++++++++-- .../resources/application-test.properties | 1 + .../server/queue/TbQueueConsumer.java | 2 + .../AbstractTbQueueConsumerTemplate.java | 5 ++ .../queue/memory/InMemoryTbQueueConsumer.java | 5 ++ .../dao/housekeeper/CleanUpService.java | 2 +- 9 files changed, 121 insertions(+), 27 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java index 63e8bb6a75..589c570144 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -53,6 +54,7 @@ import java.util.stream.Collectors; @TbCoreComponent @Service @Slf4j +@ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) public class DefaultHousekeeperService implements HousekeeperService { private final Map> taskProcessors; @@ -136,9 +138,7 @@ public class DefaultHousekeeperService implements HousekeeperService { } if (log.isDebugEnabled()) { - log.debug("[{}] {} task {}", task.getTenantId(), - msg.getTask().getErrorsCount() == 0 ? "Processing" : "Reprocessing", - msg.getTask().getValue()); + log.debug("[{}] {} task {}", task.getTenantId(), isNew(msg.getTask()) ? "Processing" : "Reprocessing", msg.getTask().getValue()); } try { Future future = executor.submit(() -> { @@ -146,7 +146,9 @@ public class DefaultHousekeeperService implements HousekeeperService { return null; }); future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); + statsService.reportProcessed(task.getTaskType(), msg); + log.debug("[{}] Successfully {} task {}", task.getTenantId(), isNew(msg.getTask()) ? "processed" : "reprocessed", msg.getTask().getValue()); } catch (InterruptedException e) { throw e; } catch (Throwable e) { @@ -160,6 +162,7 @@ public class DefaultHousekeeperService implements HousekeeperService { task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(msg, error); + statsService.reportFailure(task.getTaskType(), msg); notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() .task(task) @@ -185,12 +188,19 @@ public class DefaultHousekeeperService implements HousekeeperService { .build()), null); } + private boolean isNew(HousekeeperTaskProto task) { + return task.getErrorsCount() == 0; + } + @PreDestroy - private void stop() { - log.info("Stopped Housekeeper service"); + private void stop() throws Exception { stopped = true; consumer.unsubscribe(); - consumerExecutor.shutdownNow(); + consumerExecutor.shutdown(); + if (!consumerExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + consumerExecutor.shutdownNow(); + } + log.info("Stopped Housekeeper service"); } } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 4d361cfba4..c37496ff49 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; @@ -47,16 +49,18 @@ import java.util.concurrent.TimeUnit; @TbCoreComponent @Service @Slf4j +@ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) public class HousekeeperReprocessingService { private final DefaultHousekeeperService housekeeperService; private final PartitionService partitionService; private final TbCoreQueueFactory queueFactory; - private final TbQueueProducerProvider producerProvider; + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; - @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:15}") // fixme: to 5 minutes + @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:300}") private int startDelay; - @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:30}") // fixme: to 30 minutes or 1 hour + @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:3600}") private int reprocessingDelay; @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") private int maxReprocessingAttempts; @@ -74,7 +78,8 @@ public class HousekeeperReprocessingService { this.housekeeperService = housekeeperService; this.partitionService = partitionService; this.queueFactory = queueFactory; - this.producerProvider = producerProvider; + this.producer = producerProvider.getHousekeeperReprocessingMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); } @PostConstruct @@ -101,7 +106,13 @@ public class HousekeeperReprocessingService { while (!stopped) { try { List> msgs = consumer.poll(pollInterval); - if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately + if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { + // it's not time yet to process the message + if (!consumer.isCommitSupported()) { + for (TbProtoQueueMsg msg : msgs) { + producer.send(submitTpi, new TbProtoQueueMsg<>(msg.getKey(), msg.getValue()), null); + } + } break; } @@ -114,7 +125,6 @@ public class HousekeeperReprocessingService { } catch (Throwable e) { log.error("Unexpected error during message reprocessing [{}]", msg, e); submitForReprocessing(msg.getValue(), e); - // fixme: msgs are duplicated } } consumer.commit(); @@ -150,16 +160,18 @@ public class HousekeeperReprocessingService { .build(); log.trace("Submitting for reprocessing: {}", msg); - var producer = producerProvider.getHousekeeperReprocessingMsgProducer(); - TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); // reprocessing topic has single partition, so we don't care about the msg key + producer.send(submitTpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); // reprocessing topic has single partition, so we don't care about the msg key } @PreDestroy - private void stop() { + private void stop() throws Exception { stopped = true; scheduler.shutdownNow(); - consumerExecutor.shutdownNow(); + consumerExecutor.shutdown(); + if (!consumerExecutor.awaitTermination(10, TimeUnit.SECONDS)) { + consumerExecutor.shutdownNow(); + } + log.info("Stopped Housekeeper reprocessing service"); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4ced0655f0..a156762c71 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1427,8 +1427,10 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" - housekeeper-reprocessing: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1" + # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days + housekeeper-reprocessing: "retention.ms:7776000000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1585,12 +1587,13 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: + enabled: "${TB_HOUSEKEEPER_ENABLED:true}" topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" - reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:15}" # fixme: to 5 minutes - task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:30}" # fixme: to 30 minutes or 1 hour + reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:300}" + task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:3600}" max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}" vc: diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index b132cf34fd..a0564d3bcb 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatcher; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -56,33 +58,46 @@ import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.event.EventService; -import org.thingsboard.server.dao.housekeeper.HousekeeperService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.service.housekeeper.processor.TelemetryDeletionTaskProcessor; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @TestPropertySource(properties = { - "transport.http.enabled=true" + "queue.core.housekeeper.enabled=true", + "transport.http.enabled=true", + "queue.core.housekeeper.reprocessing-start-delay-sec=1", + "queue.core.housekeeper.task-reprocessing-delay-sec=2", + "queue.core.housekeeper.poll-interval-ms=1000" }) public class HousekeeperServiceTest extends AbstractControllerTest { @SpyBean - private HousekeeperService housekeeperService; + private DefaultHousekeeperService housekeeperService; + @SpyBean + private HousekeeperReprocessingService housekeeperReprocessingService; @Autowired private EventService eventService; @Autowired @@ -93,6 +108,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private RuleChainService ruleChainService; @Autowired private AlarmService alarmService; + @SpyBean + private TelemetryDeletionTaskProcessor telemetryDeletionTaskProcessor; private TenantId tenantId; @@ -112,7 +129,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenDeviceIsDeleted_thenCleanUpRelatedData() throws Exception { - Device device = createDevice("test", "test"); + Device device = createDevice("wekfwepf", "wekfwepf"); createRelatedData(device.getId()); doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); @@ -143,7 +160,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenUserIsDeleted_thenCleanUpRelatedData() throws Exception { - Device device = createDevice("test", "test"); + Device device = createDevice("vneoruvhwe", "vneoruvhwe"); UserId userId = customerUserId; createRelatedData(userId); Alarm alarm = Alarm.builder() @@ -172,7 +189,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { createRelatedData(tenantId); - Device device = createDevice("test", "test"); + Device device = createDevice("oi324rujoi", "oi324rujoi"); createRelatedData(device.getId()); RuleChainMetaData ruleChainMetaData = createRuleChain(); @@ -199,6 +216,45 @@ public class HousekeeperServiceTest extends AbstractControllerTest { }); } + @Test + public void whenTaskProcessingFails_thenReprocessUntilSuccessful() throws Exception { + TimeoutException error = new TimeoutException("Test timeout"); + doThrow(error).when(telemetryDeletionTaskProcessor).process(any()); + + Device device = createDevice("vep9ruv32", "vep9ruv32"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + int attempts = 3; + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + verify(housekeeperService).processTask(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, + task -> task.getErrorsCount() == 0))); + + for (int i = 1; i <= attempts; i++) { + int attempt = i; + verify(housekeeperReprocessingService).submitForReprocessing(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, + task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); + verify(housekeeperService).processTask(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, + task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt))); + } + }); + + assertThat(getTimeseriesHistory(device.getId())).isNotEmpty(); + doCallRealMethod().when(telemetryDeletionTaskProcessor).process(any()); // fixing the code + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(getTimeseriesHistory(device.getId())).isEmpty(); + }); + } + + private ArgumentMatcher verifyTaskSubmission(EntityId entityId, HousekeeperTaskType taskType, + Predicate additionalCheck) { + return msg -> { + HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); + return task.getEntityId().equals(entityId) && task.getTaskType() == taskType && additionalCheck.test(msg.getTask()); + }; + } + private void createRelatedData(EntityId entityId) throws Exception { createTelemetry(entityId); for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 92959e4552..5e4900fdc1 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -54,3 +54,4 @@ sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 server.log_controller_error_stack_trace=false +queue.core.housekeeper.enabled=false diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 51456ab0e2..3747e6d026 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -36,6 +36,8 @@ public interface TbQueueConsumer { void commit(); + boolean isCommitSupported(); + boolean isStopped(); List getFullTopicNames(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 531e30b343..9105c829bc 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -201,6 +201,11 @@ public abstract class AbstractTbQueueConsumerTemplate i return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } + @Override + public boolean isCommitSupported() { + return true; + } + protected boolean isLongPollingSupported() { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index de3833d124..269924087b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -114,4 +114,9 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } + @Override + public boolean isCommitSupported() { + return false; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 13a1e1cb09..015917e592 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -58,7 +58,7 @@ public class CleanUpService { } public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { - log.trace("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); + log.debug("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); housekeeperService.ifPresent(housekeeperService -> { From 82476b629c0c6ac3ad31b627aa5734cf66d1f50d Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 13 Mar 2024 18:34:27 +0200 Subject: [PATCH 16/72] Max reprocessing attempts for Housekeeper task --- .../HousekeeperReprocessingService.java | 45 ++++++++++++-- .../housekeeper/HousekeeperServiceTest.java | 58 +++++++++++++++---- 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index c37496ff49..5c577dd39f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -45,6 +45,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; @TbCoreComponent @Service @@ -70,6 +71,7 @@ public class HousekeeperReprocessingService { private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler")); + protected AtomicInteger cycle = new AtomicInteger(); private boolean stopped; public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, @@ -86,6 +88,7 @@ public class HousekeeperReprocessingService { private void init() { scheduler.scheduleWithFixedDelay(() -> { try { + cycle.incrementAndGet(); startReprocessing(); } catch (Throwable e) { log.error("Unexpected error during reprocessing", e); @@ -109,8 +112,9 @@ public class HousekeeperReprocessingService { if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // it's not time yet to process the message if (!consumer.isCommitSupported()) { + // resubmitting consumed messages if committing is not supported (for in-memory queue) for (TbProtoQueueMsg msg : msgs) { - producer.send(submitTpi, new TbProtoQueueMsg<>(msg.getKey(), msg.getValue()), null); + submit(msg.getKey(), msg.getValue()); } } break; @@ -119,7 +123,7 @@ public class HousekeeperReprocessingService { for (TbProtoQueueMsg msg : msgs) { log.trace("Reprocessing task: {}", msg); try { - housekeeperService.processTask(msg.getValue()); + reprocessTask(msg.getValue()); } catch (InterruptedException e) { return; } catch (Throwable e) { @@ -144,7 +148,26 @@ public class HousekeeperReprocessingService { }); } - // todo: dead letter queue if attempts count exceeds the configured maximum + private void reprocessTask(ToHousekeeperServiceMsg msg) throws Exception { + int attempt = msg.getTask().getAttempt(); + if (attempt > maxReprocessingAttempts) { + if (cycle.get() == 1) { // only reprocessing tasks with exceeded failures on first cycle (after start-up) + log.info("Trying to reprocess task with {} failed attempts: {}", attempt, msg); + } else { + // resubmitting msg to be processed on the next service start + msg = msg.toBuilder() + .setTask(msg.getTask().toBuilder() + .setTs(getReprocessingTs()) + .build()) + .build(); + submit(UUID.randomUUID(), msg); + return; + } + } + + housekeeperService.processTask(msg); + } + public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { HousekeeperTaskProto task = msg.getTask(); @@ -155,12 +178,24 @@ public class HousekeeperReprocessingService { .setTask(task.toBuilder() .setAttempt(attempt) .clearErrors().addAllErrors(errors) - .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis((long) (reprocessingDelay * 0.8))) + .setTs(getReprocessingTs()) .build()) .build(); log.trace("Submitting for reprocessing: {}", msg); - producer.send(submitTpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); // reprocessing topic has single partition, so we don't care about the msg key + submit(UUID.randomUUID(), msg); // reprocessing topic has single partition, so we don't care about the msg key + + if (task.getAttempt() >= maxReprocessingAttempts) { + log.warn("Failed to process task in {} attempts: {}", task.getAttempt(), msg); + } + } + + private void submit(UUID key, ToHousekeeperServiceMsg msg) { + producer.send(submitTpi, new TbProtoQueueMsg<>(key, msg), null); + } + + private long getReprocessingTs() { + return System.currentTimeMillis() + TimeUnit.SECONDS.toMillis((long) (reprocessingDelay * 0.8)); // *0.8 so that msgs submitted just after finishing reprocessing are processed on the next cycle } @PreDestroy diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index a0564d3bcb..ef3ce8c786 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -21,6 +21,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; @@ -36,6 +37,7 @@ import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -81,6 +83,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -90,7 +93,8 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. "transport.http.enabled=true", "queue.core.housekeeper.reprocessing-start-delay-sec=1", "queue.core.housekeeper.task-reprocessing-delay-sec=2", - "queue.core.housekeeper.poll-interval-ms=1000" + "queue.core.housekeeper.poll-interval-ms=1000", + "queue.core.housekeeper.max-reprocessing-attempts=5" }) public class HousekeeperServiceTest extends AbstractControllerTest { @@ -170,7 +174,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { .severity(AlarmSeverity.MAJOR) .build(); alarm = doPost("/api/alarm", alarm, Alarm.class); - alarm = doPost("/api/alarm/" + alarm.getId() + "/assign/" + userId, "", Alarm.class); + AlarmId alarmId = alarm.getId(); + alarm = doPost("/api/alarm/" + alarmId + "/assign/" + userId, "", Alarm.class); assertThat(alarm.getAssigneeId()).isEqualTo(userId); assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(100)).getData()).isNotEmpty(); @@ -178,8 +183,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { verifyNoRelatedData(userId); + assertThat(alarmService.findAlarmById(tenantId, alarmId).getAssigneeId()).isNull(); }); - assertThat(alarmService.findAlarmById(tenantId, alarm.getId()).getAssigneeId()).isNull(); } @Test @@ -228,15 +233,12 @@ public class HousekeeperServiceTest extends AbstractControllerTest { int attempts = 3; await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - verify(housekeeperService).processTask(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, - task -> task.getErrorsCount() == 0))); - + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, 0); for (int i = 1; i <= attempts; i++) { int attempt = i; - verify(housekeeperReprocessingService).submitForReprocessing(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, + verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); - verify(housekeeperService).processTask(argThat(verifyTaskSubmission(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, - task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt))); + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, attempt); } }); @@ -247,11 +249,43 @@ public class HousekeeperServiceTest extends AbstractControllerTest { }); } - private ArgumentMatcher verifyTaskSubmission(EntityId entityId, HousekeeperTaskType taskType, - Predicate additionalCheck) { + @Test + public void whenReprocessingAttemptsExceeded_thenReprocessOnNextStartUp() throws Exception { + TimeoutException error = new TimeoutException("Test timeout"); + doThrow(error).when(telemetryDeletionTaskProcessor).process(any()); + + Device device = createDevice("woeifjiowejf", "woeifjiowejf"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + int maxAttempts = 5; + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + for (int i = 1; i <= maxAttempts; i++) { + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, i); + } + }); + + Mockito.clearInvocations(housekeeperService); + doCallRealMethod().when(telemetryDeletionTaskProcessor).process(any()); + TimeUnit.SECONDS.sleep(2); + verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, null))); + + housekeeperReprocessingService.cycle.set(0); // imitating start-up + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, 6); + }); + } + + private void verifyTaskProcessing(EntityId entityId, HousekeeperTaskType taskType, int expectedAttempt) throws Exception { + verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); + } + + private ArgumentMatcher getTaskMatcher(EntityId entityId, HousekeeperTaskType taskType, + Predicate additionalCheck) { return msg -> { HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); - return task.getEntityId().equals(entityId) && task.getTaskType() == taskType && additionalCheck.test(msg.getTask()); + return task.getEntityId().equals(entityId) && task.getTaskType() == taskType && (additionalCheck == null || additionalCheck.test(msg.getTask())); }; } From 22f5573e8d9360692847a8589be1b3c50dd4f030 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 13 Mar 2024 19:59:13 +0200 Subject: [PATCH 17/72] Housekeeper client for all service types --- .../HousekeeperReprocessingService.java | 18 +++--- ...erService.java => HousekeeperService.java} | 44 ++++---------- .../stats/HousekeeperStatsService.java | 8 ++- .../src/main/resources/thingsboard.yml | 23 ++++++-- .../housekeeper/HousekeeperServiceTest.java | 7 ++- .../msg/housekeeper/HousekeeperClient.java | 4 +- .../housekeeper/DefaultHousekeeperClient.java | 58 +++++++++++++++++++ .../provider/AwsSqsMonolithQueueFactory.java | 10 ++-- .../provider/AwsSqsTbCoreQueueFactory.java | 10 ++-- .../AwsSqsTbRuleEngineQueueFactory.java | 6 +- .../AwsSqsTbVersionControlQueueFactory.java | 5 ++ .../provider/AwsSqsTransportQueueFactory.java | 5 ++ .../HousekeeperClientQueueFactory.java | 26 +++++++++ .../InMemoryTbTransportQueueFactory.java | 5 ++ .../KafkaTbRuleEngineQueueFactory.java | 12 ++++ .../KafkaTbTransportQueueFactory.java | 12 ++++ .../KafkaTbVersionControlQueueFactory.java | 13 +++++ .../provider/PubSubMonolithQueueFactory.java | 12 ++-- .../provider/PubSubTbCoreQueueFactory.java | 12 ++-- .../PubSubTbRuleEngineQueueFactory.java | 7 ++- .../PubSubTbVersionControlQueueFactory.java | 5 ++ .../provider/PubSubTransportQueueFactory.java | 5 ++ .../RabbitMqMonolithQueueFactory.java | 12 ++-- .../provider/RabbitMqTbCoreQueueFactory.java | 12 ++-- .../RabbitMqTbRuleEngineQueueFactory.java | 5 ++ .../RabbitMqTbVersionControlQueueFactory.java | 5 ++ .../RabbitMqTransportQueueFactory.java | 5 ++ .../ServiceBusMonolithQueueFactory.java | 12 ++-- .../ServiceBusTbCoreQueueFactory.java | 12 ++-- .../ServiceBusTbRuleEngineQueueFactory.java | 7 ++- ...erviceBusTbVersionControlQueueFactory.java | 5 ++ .../ServiceBusTransportQueueFactory.java | 5 ++ .../queue/provider/TbCoreQueueFactory.java | 4 +- .../provider/TbCoreQueueProducerProvider.java | 7 --- .../provider/TbQueueProducerProvider.java | 2 - .../TbRuleEngineProducerProvider.java | 12 ++-- .../provider/TbRuleEngineQueueFactory.java | 2 +- .../provider/TbTransportQueueFactory.java | 2 +- .../TbTransportQueueProducerProvider.java | 11 ++-- .../TbVersionControlProducerProvider.java | 13 ++--- .../TbVersionControlQueueFactory.java | 2 +- .../dao/housekeeper/CleanUpService.java | 39 ++++--------- .../dao/service/AbstractServiceTest.java | 18 ++++++ 43 files changed, 340 insertions(+), 159 deletions(-) rename application/src/main/java/org/thingsboard/server/service/housekeeper/{DefaultHousekeeperService.java => HousekeeperService.java} (78%) rename dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java => common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java (88%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 5c577dd39f..8b1dece58e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -32,7 +32,6 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.PostConstruct; @@ -53,11 +52,11 @@ import java.util.concurrent.atomic.AtomicInteger; @ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) public class HousekeeperReprocessingService { - private final DefaultHousekeeperService housekeeperService; + private final HousekeeperService housekeeperService; private final PartitionService partitionService; private final TbCoreQueueFactory queueFactory; - private final TbQueueProducer> producer; - private final TopicPartitionInfo submitTpi; + private TbQueueProducer> producer; + private TopicPartitionInfo submitTpi; @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:300}") private int startDelay; @@ -74,18 +73,19 @@ public class HousekeeperReprocessingService { protected AtomicInteger cycle = new AtomicInteger(); private boolean stopped; - public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService, - PartitionService partitionService, TbCoreQueueFactory queueFactory, - TbQueueProducerProvider producerProvider) { + public HousekeeperReprocessingService(@Lazy HousekeeperService housekeeperService, + PartitionService partitionService, + TbCoreQueueFactory queueFactory) { this.housekeeperService = housekeeperService; this.partitionService = partitionService; this.queueFactory = queueFactory; - this.producer = producerProvider.getHousekeeperReprocessingMsgProducer(); - this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); } @PostConstruct private void init() { + producer = queueFactory.createHousekeeperReprocessingMsgProducer(); + submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + scheduler.scheduleWithFixedDelay(() -> { try { cycle.incrementAndGet(); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java rename to application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 589c570144..4c0c0e72d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -26,15 +26,11 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.housekeeper.HousekeeperService; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; -import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; -import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; @@ -43,6 +39,7 @@ import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; import javax.annotation.PreDestroy; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -55,16 +52,14 @@ import java.util.stream.Collectors; @Service @Slf4j @ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) -public class DefaultHousekeeperService implements HousekeeperService { +public class HousekeeperService { private final Map> taskProcessors; private final HousekeeperReprocessingService reprocessingService; - private final HousekeeperStatsService statsService; + private final Optional statsService; private final NotificationRuleProcessor notificationRuleProcessor; private final TbQueueConsumer> consumer; - private final TbQueueProducer> producer; - private final TopicPartitionInfo submitTpi; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); @@ -76,18 +71,15 @@ public class DefaultHousekeeperService implements HousekeeperService { private boolean stopped; - public DefaultHousekeeperService(HousekeeperReprocessingService reprocessingService, - TbCoreQueueFactory queueFactory, - TbQueueProducerProvider producerProvider, - HousekeeperStatsService statsService, - NotificationRuleProcessor notificationRuleProcessor, - @Lazy List> taskProcessors) { + public HousekeeperService(HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + Optional statsService, + NotificationRuleProcessor notificationRuleProcessor, + @Lazy List> taskProcessors) { this.reprocessingService = reprocessingService; this.statsService = statsService; this.notificationRuleProcessor = notificationRuleProcessor; this.consumer = queueFactory.createHousekeeperMsgConsumer(); - this.producer = producerProvider.getHousekeeperMsgProducer(); - this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); } @@ -147,7 +139,7 @@ public class DefaultHousekeeperService implements HousekeeperService { }); future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); - statsService.reportProcessed(task.getTaskType(), msg); + statsService.ifPresent(statsService -> statsService.reportProcessed(task.getTaskType(), msg)); log.debug("[{}] Successfully {} task {}", task.getTenantId(), isNew(msg.getTask()) ? "processed" : "reprocessed", msg.getTask().getValue()); } catch (InterruptedException e) { throw e; @@ -163,7 +155,7 @@ public class DefaultHousekeeperService implements HousekeeperService { task.getTaskType(), msg.getTask().getAttempt(), task, error); reprocessingService.submitForReprocessing(msg, error); - statsService.reportFailure(task.getTaskType(), msg); + statsService.ifPresent(statsService -> statsService.reportFailure(task.getTaskType(), msg)); notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() .task(task) .error(error) @@ -172,22 +164,6 @@ public class DefaultHousekeeperService implements HousekeeperService { } } - @Override - public void submitTask(HousekeeperTask task) { - log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); - /* - * using msg key as entity id so that msgs related to certain entity are pushed to same partition, - * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order - * */ - producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() - .setTask(HousekeeperTaskProto.newBuilder() - .setValue(JacksonUtil.toString(task)) - .setTs(task.getTs()) - .setAttempt(0) - .build()) - .build()), null); - } - private boolean isNew(HousekeeperTaskProto task) { return task.getErrorsCount() == 0; } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 3f6e88a299..400bc36479 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.housekeeper.stats; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; -import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import java.util.ArrayList; @@ -31,11 +32,11 @@ import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Service @Slf4j +@ConditionalOnProperty(name = "queue.core.housekeeper.stats.enabled", havingValue = "true", matchIfMissing = true) public class HousekeeperStatsService { private final Map stats = new EnumMap<>(HousekeeperTaskType.class); @@ -46,7 +47,8 @@ public class HousekeeperStatsService { } } - @Scheduled(initialDelay = 60, fixedDelay = 60, timeUnit = TimeUnit.SECONDS) + @Scheduled(initialDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}", + fixedDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}") private void reportStats() { String statsStr = stats.values().stream().map(stats -> { String countersStr = stats.getCounters().stream() diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a156762c71..5baa155ae9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1401,9 +1401,11 @@ queue: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" tb_housekeeper: + # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one - key: max.poll.records value: "1" tb_housekeeper.reprocessing: + # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one - key: max.poll.records value: "1" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" @@ -1428,9 +1430,9 @@ queue: # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Housekeeper tasks topic - housekeeper: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1" - # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days - housekeeper-reprocessing: "retention.ms:7776000000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1" + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time + housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1587,14 +1589,27 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: + # Enabled/disable Housekeeping service, which does cleanup of telemetry, attributes, events, etc. enabled: "${TB_HOUSEKEEPER_ENABLED:true}" + # Topic name for Housekeeper tasks topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + # Topic name for Housekeeper tasks to be reprocessed reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + # Poll interval for topics related to Housekeeper poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + # Delay in seconds for starting tasks reprocessing after start-up reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:300}" + # Period in seconds of tasks reprocessing task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:3600}" - max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:30}" + # Maximum amount of task reprocessing attempts. After exceeding, the task will be ignored until the next service start-up + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" + stats: + # Enable/disable statistics for Housekeeper + enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" + # Statistics printing interval for Housekeeper + print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" vc: # Default topic name for Kafka, RabbitMQ, etc. diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index ef3ce8c786..daf363467d 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -56,6 +56,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; @@ -99,7 +100,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. public class HousekeeperServiceTest extends AbstractControllerTest { @SpyBean - private DefaultHousekeeperService housekeeperService; + private HousekeeperService housekeeperService; + @SpyBean + private HousekeeperClient housekeeperClient; @SpyBean private HousekeeperReprocessingService housekeeperReprocessingService; @Autowired @@ -300,7 +303,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private void verifyNoRelatedData(EntityId entityId) throws Exception { List expectedTaskTypes = List.of(HousekeeperTaskType.DELETE_TELEMETRY, HousekeeperTaskType.DELETE_ATTRIBUTES, HousekeeperTaskType.DELETE_EVENTS, HousekeeperTaskType.DELETE_ENTITY_ALARMS); for (HousekeeperTaskType taskType : expectedTaskTypes) { - verify(housekeeperService).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); + verify(housekeeperClient).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); } assertThat(getLatestTelemetry(entityId)).isNull(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java b/common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java similarity index 88% rename from dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java index e3c0800f69..e2a3af9242 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/HousekeeperService.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper; +package org.thingsboard.server.common.msg.housekeeper; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; -public interface HousekeeperService { +public interface HousekeeperClient { void submitTask(HousekeeperTask task); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java new file mode 100644 index 0000000000..b766f3826b --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.housekeeper; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; + +@Service +@Slf4j +public class DefaultHousekeeperClient implements HousekeeperClient { + + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; + + public DefaultHousekeeperClient(TbQueueProducerProvider producerProvider) { + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + } + + @Override + public void submitTask(HousekeeperTask task) { + log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + /* + * using msg key as entity id so that msgs related to certain entity are pushed to same partition, + * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order + * */ + producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() + .setTask(TransportProtos.HousekeeperTaskProto.newBuilder() + .setValue(JacksonUtil.toString(task)) + .setTs(task.getTs()) + .setAttempt(0) + .build()) + .build()), null); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 1217186410..e60594c4a1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -229,22 +229,24 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 1a4f92fcd3..eb3706657b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -208,22 +208,24 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 6d48ceb1b6..1653955ae9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -33,8 +33,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -159,6 +159,10 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } @PreDestroy private void destroy() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java index 3aec2e3a39..6801703231 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java @@ -80,6 +80,11 @@ public class AwsSqsTbVersionControlQueueFactory implements TbVersionControlQueue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index c27ad4e092..96a9686d13 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -131,6 +131,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java new file mode 100644 index 0000000000..79c38a04e9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface HousekeeperClientQueueFactory { + + TbQueueProducer> createHousekeeperMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index f08a822466..4ee79aef60 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -120,4 +120,9 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 63798f95a1..869f364970 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -72,6 +72,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin jsExecutorResponseAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -97,6 +98,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.jsExecutorResponseAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorResponseConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -230,6 +232,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 7cecc94290..4140925775 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -66,6 +66,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueAdmin transportApiRequestAdmin; private final TbQueueAdmin transportApiResponseAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, @@ -90,6 +91,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { this.transportApiRequestAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiRequestConfigs()); this.transportApiResponseAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiResponseConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -173,6 +175,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-transport-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java index e39da6bdfc..95086e04b4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; @@ -51,6 +52,7 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF private final TbQueueAdmin coreAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, @@ -69,6 +71,7 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @@ -105,6 +108,16 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-vc-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index e6a23b8068..0f2485b6a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -41,8 +41,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -230,22 +230,24 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 1b52a2ba42..00a6d83bdd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -39,8 +39,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -201,22 +201,24 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 070384e60e..d0192d3094 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -36,8 +36,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -161,6 +161,11 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java index ed40a43deb..509cf371b8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 3323308bef..d91dd91873 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -131,6 +131,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index fa89956b64..78b60bca28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -41,8 +41,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -227,22 +227,24 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index b1f3de672f..db3414b62d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -38,8 +38,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -200,22 +200,24 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 929ea8298c..556aa968c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -160,6 +160,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java index bd14db38b4..a614df3fdb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class RabbitMqTbVersionControlQueueFactory implements TbVersionControlQue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index d5fb7bd0e2..389a6d11ee 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -132,6 +132,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 5f4be98ca0..a1f1c7adc8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -45,8 +45,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -226,22 +226,24 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index 579fba272c..854295cb6b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -44,8 +44,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -201,22 +201,24 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueProducer> createHousekeeperMsgProducer() { - return null; + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } @Override public TbQueueConsumer> createHousekeeperMsgConsumer() { - return null; + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return null; + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); } @Override public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return null; + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @PreDestroy diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index b09ab1d30d..066e84e780 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -41,8 +41,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -160,6 +160,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java index 2e5e5c4bc3..bed6c427c9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class ServiceBusTbVersionControlQueueFactory implements TbVersionControlQ ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index d782a1ff9e..0db62ac90c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -133,6 +133,11 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 866eb931f2..20a67c2e09 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -37,7 +37,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -132,8 +132,6 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { */ TbQueueProducer> createVersionControlMsgProducer(); - TbQueueProducer> createHousekeeperMsgProducer(); - TbQueueConsumer> createHousekeeperMsgConsumer(); TbQueueProducer> createHousekeeperReprocessingMsgProducer(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index e6bc8f0484..aa2c55fca1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -43,7 +43,6 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; private TbQueueProducer> toHousekeeper; - private TbQueueProducer> toHousekeeperReprocessing; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -59,7 +58,6 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); - this.toHousekeeperReprocessing = tbQueueProvider.createHousekeeperReprocessingMsgProducer(); } @Override @@ -102,9 +100,4 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } - @Override - public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { - return toHousekeeperReprocessing; - } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index ff56d2e8be..a752301cd8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -82,6 +82,4 @@ public interface TbQueueProducerProvider { TbQueueProducer> getHousekeeperMsgProducer(); - TbQueueProducer> getHousekeeperReprocessingMsgProducer(); - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index ee6700906c..748c48e52a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -40,6 +41,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -53,6 +55,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -91,13 +94,8 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { } @Override - public TbQueueProducer> getHousekeeperMsgProducer() { - return null; // fixme - } - - @Override - public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index cd4575af6a..f23c7e47f3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -32,7 +32,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index 909089f175..c1a0a0fd24 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -26,7 +26,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 869c588fc7..c0472ba6f0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -38,6 +39,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider private TbQueueProducer> toTbCore; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -49,6 +51,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -87,12 +90,8 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider } @Override - public TbQueueProducer> getHousekeeperMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; } - @Override - public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Transport!"); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index 3f4bd209e4..c4e8b697a3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -36,6 +37,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider private final TbVersionControlQueueFactory tbQueueProvider; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -45,6 +47,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public void init() { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -54,7 +57,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider @Override public TbQueueProducer> getRuleEngineMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); } @Override @@ -83,12 +86,8 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider } @Override - public TbQueueProducer> getHousekeeperMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; } - @Override - public TbQueueProducer> getHousekeeperReprocessingMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java index 168a466015..3917df24cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -25,7 +25,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push notifications to other instances of TB Core Service diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 015917e592..56946c4705 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -17,8 +17,6 @@ package org.thingsboard.server.dao.housekeeper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; @@ -26,23 +24,17 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.entity.EntityDaoService; -import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.relation.RelationService; -import java.util.Optional; - @Component @RequiredArgsConstructor @Slf4j public class CleanUpService { - private final Optional housekeeperService; + private final HousekeeperClient housekeeperClient; private final RelationService relationService; - @Autowired - @Lazy - private EntityServiceRegistry entityServiceRegistry; @TransactionalEventListener(fallbackExecution = true) public void handleEntityDeletionEvent(DeleteEntityEvent event) { @@ -51,9 +43,7 @@ public class CleanUpService { log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityId.getEntityType(), entityId.getId()); cleanUpRelatedData(tenantId, entityId); if (entityId.getEntityType() == EntityType.USER) { - housekeeperService.ifPresent(housekeeperService -> { - housekeeperService.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); - }); + housekeeperClient.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } } @@ -61,25 +51,16 @@ public class CleanUpService { log.debug("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); - housekeeperService.ifPresent(housekeeperService -> { - housekeeperService.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperService.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); - }); + housekeeperClient.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + housekeeperClient.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + housekeeperClient.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + housekeeperClient.submitTask(HousekeeperTask.deleteEntityAlarms(tenantId, entityId)); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { - housekeeperService.ifPresentOrElse(housekeeperService -> { - for (EntityType entityType : entityTypes) { - housekeeperService.submitTask(HousekeeperTask.deleteEntities(tenantId, entityType)); - } - }, () -> { - for (EntityType entityType : entityTypes) { - EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); - entityService.deleteByTenantId(tenantId); - } - }); + for (EntityType entityType : entityTypes) { + housekeeperClient.submitTask(HousekeeperTask.deleteEntities(tenantId, entityType)); + } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index e8232af692..f222c05b60 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTra import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -51,6 +53,9 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; import org.thingsboard.server.dao.audit.AuditLogLevelMask; import org.thingsboard.server.dao.audit.AuditLogLevelProperties; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.dao.tenant.TenantService; import java.io.IOException; @@ -77,6 +82,9 @@ public abstract class AbstractServiceTest { @Autowired protected TenantService tenantService; + @Autowired + protected EntityServiceRegistry entityServiceRegistry; + protected TenantId tenantId; @Before @@ -126,6 +134,16 @@ public abstract class AbstractServiceTest { return new AuditLogLevelFilter(props); } + @Bean + public HousekeeperClient housekeeperClient() { + return task -> { + if (task.getTaskType() == HousekeeperTaskType.DELETE_ENTITIES) { + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(((EntitiesDeletionHousekeeperTask) task).getEntityType()); + entityService.deleteByTenantId(task.getTenantId()); + } + }; + } + protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setTenantId(tenantId); From a576a73b8629378fcbebe3b39379da9029b78e91 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 13 Mar 2024 20:08:17 +0200 Subject: [PATCH 18/72] Housekeeper topic configs for all ymls --- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 5 +++++ transport/coap/src/main/resources/tb-coap-transport.yml | 5 +++++ transport/http/src/main/resources/tb-http-transport.yml | 5 +++++ transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 5 +++++ transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 5 +++++ transport/snmp/src/main/resources/tb-snmp-transport.yml | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 289baaa97a..a92a7eea2b 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -137,6 +137,8 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Core topics version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -250,6 +252,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index faa5f43065..a758a64882 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -277,6 +277,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -398,6 +400,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 49bf202200..4c8dcd0e45 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -261,6 +261,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -383,6 +385,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index b6dc7248ab..669e42bc8a 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -358,6 +358,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -477,6 +479,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 55223caf4e..c958ae2fd7 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -293,6 +293,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -414,6 +416,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index a40374e1f6..ea9f2c9ae1 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -243,6 +243,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -371,6 +373,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" From fc865252a8d2da6e62a50ca36164cf4bc7d28575 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Sat, 16 Mar 2024 15:41:48 +0200 Subject: [PATCH 19/72] Use Autowired in TenantServiceImpl --- .../validator/AssetProfileDataValidator.java | 1 - .../validator/DashboardDataValidator.java | 2 -- .../validator/DeviceDataValidator.java | 2 -- .../validator/DeviceProfileDataValidator.java | 1 - .../validator/ResourceDataValidator.java | 1 - .../validator/RuleChainDataValidator.java | 1 - .../server/dao/tenant/TenantServiceImpl.java | 36 +++++++++++++------ .../usagerecord/ApiUsageStateServiceImpl.java | 2 +- 8 files changed, 26 insertions(+), 20 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java index 7543c247e0..19a04fd867 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetProfileDataValidator.java @@ -42,7 +42,6 @@ public class AssetProfileDataValidator extends DataValidator { @Lazy private AssetProfileService assetProfileService; @Autowired - @Lazy private TenantService tenantService; @Lazy @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java index 829da4d5c7..178905f7f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; @@ -29,7 +28,6 @@ import org.thingsboard.server.dao.tenant.TenantService; public class DashboardDataValidator extends DataValidator { @Autowired - @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index a8f1bdd972..e2047dc448 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.service.validator; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -40,7 +39,6 @@ public class DeviceDataValidator extends AbstractHasOtaPackageValidator private DeviceDao deviceDao; @Autowired - @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index a06fd5edaa..23f3a6bbdb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -86,7 +86,6 @@ public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { private WidgetTypeDao widgetTypeDao; @Autowired - @Lazy private TenantService tenantService; @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java index 2f758b1b9c..222aaa3eeb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java @@ -53,7 +53,6 @@ public class RuleChainDataValidator extends DataValidator { private RuleChainService ruleChainService; @Autowired - @Lazy private TenantService tenantService; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 6ab9361b19..181989a620 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -38,9 +40,9 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.notification.NotificationSettingsService; -import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.service.validator.TenantDataValidator; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; @@ -58,16 +60,28 @@ public class TenantServiceImpl extends AbstractCachedEntityService tenantValidator; - private final TbTransactionalCache existsTenantCache; + @Autowired + private TenantDao tenantDao; + @Autowired + private TenantProfileService tenantProfileService; + @Autowired + @Lazy + private UserService userService; + @Autowired + private AssetProfileService assetProfileService; + @Autowired + private DeviceProfileService deviceProfileService; + @Lazy + @Autowired + private ApiUsageStateService apiUsageStateService; + @Autowired + private AdminSettingsService adminSettingsService; + @Autowired + private NotificationSettingsService notificationSettingsService; + @Autowired + private TenantDataValidator tenantValidator; + @Autowired + protected TbTransactionalCache existsTenantCache; @TransactionalEventListener(classes = TenantEvictEvent.class) @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 0b880b7c5f..1019481d67 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -59,7 +59,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final DataValidator apiUsageStateValidator; public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, - @Lazy TenantService tenantService, @Lazy TimeseriesService tsService, + TenantService tenantService, @Lazy TimeseriesService tsService, DataValidator apiUsageStateValidator) { this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; From b0d4faf798c2c3ab8de13ee78307095fc24c5373 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Sat, 16 Mar 2024 15:49:40 +0200 Subject: [PATCH 20/72] Fix transactional calls in TransactionalEventListener --- .../src/main/resources/thingsboard.yml | 10 +++--- .../housekeeper/DefaultHousekeeperClient.java | 36 ++++++++++++++----- .../provider/KafkaTbCoreQueueFactory.java | 24 ++++++------- .../KafkaTbRuleEngineQueueFactory.java | 5 +-- .../KafkaTbTransportQueueFactory.java | 5 +-- .../KafkaTbVersionControlQueueFactory.java | 5 +-- .../PubSubTbRuleEngineQueueFactory.java | 6 ++-- .../provider/PubSubTransportQueueFactory.java | 3 +- .../dao/housekeeper/CleanUpService.java | 18 +++++++--- 9 files changed, 73 insertions(+), 39 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5baa155ae9..651d2154de 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1401,13 +1401,15 @@ queue: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" tb_housekeeper: - # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one + # Consumer properties for Housekeeper tasks topic - key: max.poll.records - value: "1" + # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}" tb_housekeeper.reprocessing: - # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one + # Consumer properties for Housekeeper reprocessing topic - key: max.poll.records - value: "1" + # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java index b766f3826b..e9737bf90c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java @@ -19,10 +19,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @@ -33,26 +35,42 @@ public class DefaultHousekeeperClient implements HousekeeperClient { private final TbQueueProducer> producer; private final TopicPartitionInfo submitTpi; + private final TbQueueCallback submitCallback; public DefaultHousekeeperClient(TbQueueProducerProvider producerProvider) { this.producer = producerProvider.getHousekeeperMsgProducer(); this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + this.submitCallback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("Submitted Housekeeper task"); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to submit Housekeeper task", t); + } + }; } @Override public void submitTask(HousekeeperTask task) { - log.trace("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); /* * using msg key as entity id so that msgs related to certain entity are pushed to same partition, * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order * */ - producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() - .setTask(TransportProtos.HousekeeperTaskProto.newBuilder() - .setValue(JacksonUtil.toString(task)) - .setTs(task.getTs()) - .setAttempt(0) - .build()) - .build()), null); + try { + producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() + .setTask(TransportProtos.HousekeeperTaskProto.newBuilder() + .setValue(JacksonUtil.toString(task)) + .setTs(task.getTs()) + .setAttempt(0) + .build()) + .build()), submitCallback); + } catch (Throwable t) { + log.error("Failed to submit Housekeeper task {}", task, t); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 478963d300..03c7cf1173 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -21,9 +21,9 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +39,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -309,8 +309,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return TbKafkaProducerTemplate.>builder() + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) @@ -319,21 +319,21 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createHousekeeperMsgConsumer() { - return TbKafkaConsumerTemplate.>builder() + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) - .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperAdmin) .statsService(consumerStatsService) .build(); } @Override - public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { - return TbKafkaProducerTemplate.>builder() + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) @@ -342,13 +342,13 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { } @Override - public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { - return TbKafkaConsumerTemplate.>builder() + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() .settings(kafkaSettings) .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) - .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) .admin(housekeeperReprocessingAdmin) .statsService(consumerStatsService) .build(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 869f364970..f6fd5d4619 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -233,8 +234,8 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return TbKafkaProducerTemplate.>builder() + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("tb-rule-engine-housekeeper-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 4140925775..8464c98770 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -176,8 +177,8 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return TbKafkaProducerTemplate.>builder() + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("tb-transport-housekeeper-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java index 95086e04b4..4d3ddc4925 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -19,6 +19,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -109,8 +110,8 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { - return TbKafkaProducerTemplate.>builder() + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() .settings(kafkaSettings) .clientId("tb-vc-housekeeper-producer-" + serviceInfoProvider.getServiceId()) .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index d0192d3094..2b118251ac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -25,6 +25,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -157,12 +159,12 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { + public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { + public TbQueueProducer> createHousekeeperMsgProducer() { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index d91dd91873..062bd65cad 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -132,7 +133,7 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { } @Override - public TbQueueProducer> createHousekeeperMsgProducer() { + public TbQueueProducer> createHousekeeperMsgProducer() { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 56946c4705..a2bc68b61b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.housekeeper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -36,14 +38,20 @@ public class CleanUpService { private final HousekeeperClient housekeeperClient; private final RelationService relationService; - @TransactionalEventListener(fallbackExecution = true) + @TransactionalEventListener(fallbackExecution = true) // after transaction commit + @Transactional(propagation = Propagation.NOT_SUPPORTED) public void handleEntityDeletionEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); - log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityId.getEntityType(), entityId.getId()); - cleanUpRelatedData(tenantId, entityId); - if (entityId.getEntityType() == EntityType.USER) { - housekeeperClient.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + EntityType entityType = entityId.getEntityType(); + try { + log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityType, entityId.getId()); + cleanUpRelatedData(tenantId, entityId); + if (entityType == EntityType.USER) { + housekeeperClient.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + } + } catch (Throwable e) { + log.error("[{}][{}][{}] Failed to handle entity deletion event", tenantId, entityType, entityId.getId(), e); } } From a6719efee58042153f3caa3e592d828851121e9f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 18 Mar 2024 12:24:58 +0200 Subject: [PATCH 21/72] Housekeeper: alarms deletion --- .../entitiy/AbstractTbEntityService.java | 12 ----- .../entitiy/asset/DefaultTbAssetService.java | 1 - .../device/DefaultTbDeviceService.java | 1 - .../housekeeper/HousekeeperService.java | 3 +- ....java => AlarmsDeletionTaskProcessor.java} | 20 ++++--- .../service/ttl/AlarmsCleanUpService.java | 1 - .../housekeeper/HousekeeperServiceTest.java | 54 ++++++++++++++++++- .../server/dao/alarm/AlarmService.java | 4 +- .../data/housekeeper/HousekeeperTask.java | 4 +- .../data/housekeeper/HousekeeperTaskType.java | 4 +- .../server/common/data/page/PageData.java | 2 + .../server/dao/alarm/AlarmDao.java | 4 +- .../server/dao/alarm/BaseAlarmService.java | 38 ++++++++++--- .../dao/housekeeper/CleanUpService.java | 2 +- .../dao/sql/alarm/EntityAlarmRepository.java | 4 +- .../server/dao/sql/alarm/JpaAlarmDao.java | 9 +++- 16 files changed, 123 insertions(+), 40 deletions(-) rename application/src/main/java/org/thingsboard/server/service/housekeeper/processor/{EntityAlarmsDeletionTaskProcessor.java => AlarmsDeletionTaskProcessor.java} (57%) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index ff8dd52ec5..9af65c0965 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -25,15 +25,10 @@ import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.alarm.AlarmInfo; -import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.edge.EdgeService; @@ -72,13 +67,6 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; - protected void removeAlarmsByEntityId(TenantId tenantId, EntityId entityId) { - PageData alarms = - alarmService.findAlarms(tenantId, new AlarmQuery(entityId, new TimePageLink(Integer.MAX_VALUE), null, null, null, false)); - - alarms.getData().stream().map(AlarmInfo::getId).forEach(alarmId -> alarmService.delAlarm(tenantId, alarmId)); - } - protected T checkNotNull(T reference) throws ThingsboardException { return checkNotNull(reference, "Requested item wasn't found!"); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 58940318be..e30f6f9922 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -78,7 +78,6 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb TenantId tenantId = asset.getTenantId(); AssetId assetId = asset.getId(); try { - removeAlarmsByEntityId(tenantId, assetId); assetService.deleteAsset(tenantId, assetId); notificationEntityService.logEntityAction(tenantId, assetId, asset, asset.getCustomerId(), actionType, user, assetId.toString()); tbClusterService.broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index 8d43a43cbc..f7d11e88d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -97,7 +97,6 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T TenantId tenantId = device.getTenantId(); DeviceId deviceId = device.getId(); try { - removeAlarmsByEntityId(tenantId, deviceId); deviceService.deleteDevice(tenantId, deviceId); notificationEntityService.notifyDeleteDevice(tenantId, deviceId, device.getCustomerId(), device, user, deviceId.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 4c0c0e72d5..fa59eae4c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -130,7 +130,7 @@ public class HousekeeperService { } if (log.isDebugEnabled()) { - log.debug("[{}] {} task {}", task.getTenantId(), isNew(msg.getTask()) ? "Processing" : "Reprocessing", msg.getTask().getValue()); + log.debug("[{}] {} {}", task.getTenantId(), isNew(msg.getTask()) ? "Processing" : "Reprocessing", task.getDescription()); } try { Future future = executor.submit(() -> { @@ -140,7 +140,6 @@ public class HousekeeperService { future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); statsService.ifPresent(statsService -> statsService.reportProcessed(task.getTaskType(), msg)); - log.debug("[{}] Successfully {} task {}", task.getTenantId(), isNew(msg.getTask()) ? "processed" : "reprocessed", msg.getTask().getValue()); } catch (InterruptedException e) { throw e; } catch (Throwable e) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java rename to application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index 0827ce2441..2d7cf690d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntityAlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -16,27 +16,35 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.dao.alarm.AlarmService; @Component @RequiredArgsConstructor -public class EntityAlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { +@Slf4j +public class AlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { private final AlarmService alarmService; @Override public void process(HousekeeperTask task) throws Exception { - alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); - // fixme: do we need to remove alarms by originator ??? - // fixme: why alarm comments are not deleted ?? + EntityType entityType = task.getEntityId().getEntityType(); + if (entityType == EntityType.DEVICE || entityType == EntityType.ASSET) { + int count = alarmService.deleteAlarmsByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} alarms", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), count); + } else { + int count = alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} entity alarms", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), count); + } } @Override public HousekeeperTaskType getTaskType() { - return HousekeeperTaskType.DELETE_ENTITY_ALARMS; + return HousekeeperTaskType.DELETE_ALARMS; } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 76de56cf13..c0f3158c61 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -56,7 +56,6 @@ public class AlarmsCleanUpService { private final TenantService tenantService; private final AlarmDao alarmDao; private final AlarmService alarmService; - private final RelationService relationService; private final EntityActionService entityActionService; private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index daf363467d..1204e36184 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -32,12 +32,17 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -52,15 +57,19 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.msg.TbNodeConnectionType; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -115,6 +124,10 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private RuleChainService ruleChainService; @Autowired private AlarmService alarmService; + @Autowired + private AlarmDao alarmDao; + @Autowired + private RelationService relationService; @SpyBean private TelemetryDeletionTaskProcessor telemetryDeletionTaskProcessor; @@ -200,6 +213,11 @@ public class HousekeeperServiceTest extends AbstractControllerTest { Device device = createDevice("oi324rujoi", "oi324rujoi"); createRelatedData(device.getId()); + Asset asset = createAsset(); + createRelatedData(asset.getId()); + createRelation(device.getId(), asset.getId()); + createAlarm(device.getId(), asset.getId()); + RuleChainMetaData ruleChainMetaData = createRuleChain(); RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId(); @@ -216,6 +234,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { verifyNoRelatedData(device.getId()); + verifyNoRelatedData(asset.getId()); verifyNoRelatedData(ruleNode1Id); verifyNoRelatedData(ruleNode2Id); verifyNoRelatedData(ruleChainId); @@ -301,7 +320,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } private void verifyNoRelatedData(EntityId entityId) throws Exception { - List expectedTaskTypes = List.of(HousekeeperTaskType.DELETE_TELEMETRY, HousekeeperTaskType.DELETE_ATTRIBUTES, HousekeeperTaskType.DELETE_EVENTS, HousekeeperTaskType.DELETE_ENTITY_ALARMS); + List expectedTaskTypes = List.of(HousekeeperTaskType.DELETE_TELEMETRY, HousekeeperTaskType.DELETE_ATTRIBUTES, HousekeeperTaskType.DELETE_EVENTS, HousekeeperTaskType.DELETE_ALARMS); for (HousekeeperTaskType taskType : expectedTaskTypes) { verify(housekeeperClient).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); } @@ -312,6 +331,11 @@ public class HousekeeperServiceTest extends AbstractControllerTest { assertThat(getAttribute(entityId, scope, scope + ATTRIBUTE_KEY)).isNull(); } assertThat(getEvents(entityId)).isEmpty(); + assertThat(alarmDao.findEntityAlarmRecordsByEntityId(tenantId, entityId)).isEmpty(); + assertThat(alarmService.findAlarms(tenantId, AlarmQuery.builder().pageLink(new TimePageLink(100)).build()).getData()) + .filteredOn(alarm -> alarm.getOriginator().equals(entityId)).isEmpty(); + assertThat(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); + assertThat(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); } private void createAttribute(EntityId entityId, String scope, String key) throws Exception { @@ -336,6 +360,27 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } + private void createRelation(DeviceId to, AssetId from) { + EntityRelation relation = new EntityRelation(from, to, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON); + relationService.saveRelation(tenantId, relation); + } + + private void createAlarm(DeviceId deviceId, EntityId propagatedEntityId) { + Alarm alarm = doPost("/api/alarm", Alarm.builder() + .tenantId(tenantId) + .originator(deviceId) + .severity(AlarmSeverity.CRITICAL) + .type("test alarm for " + deviceId) + .propagate(true) + .build(), Alarm.class); + + List entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId()); + assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType())); + assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType())); + assertThat(alarmService.findAlarms(tenantId, AlarmQuery.builder().pageLink(new TimePageLink(100)).build()).getData()) + .filteredOn(a -> a.getOriginator().equals(deviceId)).isNotEmpty(); + } + private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { return timeseriesService.findLatest(tenantId, entityId, HousekeeperServiceTest.TELEMETRY_KEY).get().orElse(null); } @@ -355,6 +400,13 @@ public class HousekeeperServiceTest extends AbstractControllerTest { .collect(Collectors.toList()); } + private Asset createAsset() { + Asset asset = new Asset(); + asset.setName("test"); + asset.setType("test"); + return doPost("/api/asset", asset, Asset.class); + } + private RuleChainMetaData createRuleChain() { RuleChain ruleChain = new RuleChain(); ruleChain.setTenantId(tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 7a55348bed..5607e4ff50 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -81,6 +81,8 @@ public interface AlarmService extends EntityDaoService { void delAlarmTypes(TenantId tenantId, Set types); + int deleteAlarmsByEntityId(TenantId tenantId, EntityId entityId); + // Other API Alarm findAlarmById(TenantId tenantId, AlarmId alarmId); @@ -106,7 +108,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); + int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 44598d511c..bc41f553ad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -69,8 +69,8 @@ public class HousekeeperTask implements Serializable { return new AlarmsUnassignHousekeeperTask(user); } - public static HousekeeperTask deleteEntityAlarms(TenantId tenantId, EntityId entityId) { - return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ENTITY_ALARMS); + public static HousekeeperTask deleteAlarms(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); } public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index 4f30f94a45..eb5833b06f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -26,8 +26,8 @@ public enum HousekeeperTaskType { DELETE_ATTRIBUTES("attributes deletion"), DELETE_TELEMETRY("telemetry deletion"), DELETE_EVENTS("events deletion"), - UNASSIGN_ALARMS("alarms unassigning"), - DELETE_ENTITY_ALARMS("entity alarms deletion"); + DELETE_ALARMS("alarms deletion"), + UNASSIGN_ALARMS("alarms unassigning"); private final String description; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java index 9042e4865f..02129ea31b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.ToString; import java.io.Serializable; import java.util.Collections; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; @ApiModel @EqualsAndHashCode +@ToString public class PageData implements Serializable { public static final PageData EMPTY_PAGE_DATA = new PageData<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index c752ae706d..339c8fd988 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -84,7 +84,9 @@ public interface AlarmDao extends Dao { List findEntityAlarmRecords(TenantId tenantId, AlarmId id); - void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); + List findEntityAlarmRecordsByEntityId(TenantId tenantId, EntityId entityId); + + int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index a14c172a57..9b7004042d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; @@ -183,16 +184,23 @@ public class BaseAlarmService extends AbstractCachedEntityService alarms; + int count = 0; + do { + alarms = findAlarms(tenantId, AlarmQuery.builder() + .affectedEntityId(entityId) + .pageLink(new TimePageLink(pageLink, null, null)) + .build()); + for (AlarmInfo alarm : alarms.getData()) { + deleteAlarm(tenantId, alarm, true); + count++; + } + } while (alarms.hasNext()); + return count; + } + private List createEntityAlarmRecords(Alarm alarm) throws ExecutionException, InterruptedException { Set propagatedEntitiesSet = new LinkedHashSet<>(); propagatedEntitiesSet.add(alarm.getOriginator()); @@ -322,9 +348,9 @@ public class BaseAlarmService extends AbstractCachedEntityService findAllByEntityId(UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 9430773ff9..1483cf8a8e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -309,8 +309,13 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId) { - entityAlarmRepository.deleteByEntityId(entityId.getId()); + public List findEntityAlarmRecordsByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(entityAlarmRepository.findAllByEntityId(entityId.getId())); + } + + @Override + public int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId) { + return entityAlarmRepository.deleteByEntityId(entityId.getId()); } @Override From dfb40ecbe053796e6844e318ff0fef3f432bac48 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 18 Mar 2024 12:35:29 +0200 Subject: [PATCH 22/72] Skipped entity types list for cleaning up related data --- .../server/dao/housekeeper/CleanUpService.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 24cfe7c8f6..48531aab90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -30,6 +30,9 @@ import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.relation.RelationService; +import java.util.EnumSet; +import java.util.Set; + @Component @RequiredArgsConstructor @Slf4j @@ -38,6 +41,12 @@ public class CleanUpService { private final HousekeeperClient housekeeperClient; private final RelationService relationService; + private final Set skippedEntities = EnumSet.of( + EntityType.ALARM, EntityType.QUEUE, EntityType.TB_RESOURCE, EntityType.OTA_PACKAGE, + EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_TEMPLATE, + EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE + ); + @TransactionalEventListener(fallbackExecution = true) // after transaction commit @Transactional(propagation = Propagation.NOT_SUPPORTED) public void handleEntityDeletionEvent(DeleteEntityEvent event) { @@ -46,7 +55,9 @@ public class CleanUpService { EntityType entityType = entityId.getEntityType(); try { log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityType, entityId.getId()); - cleanUpRelatedData(tenantId, entityId); + if (!skippedEntities.contains(entityType)) { + cleanUpRelatedData(tenantId, entityId); + } if (entityType == EntityType.USER) { housekeeperClient.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } @@ -57,7 +68,6 @@ public class CleanUpService { public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { log.debug("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); - // todo: skipped entities list relationService.deleteEntityRelations(tenantId, entityId); housekeeperClient.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); housekeeperClient.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); From 302fed18883caf74d4626ba8f47d1bf00d0ff372 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 18 Mar 2024 16:55:39 +0200 Subject: [PATCH 23/72] Await tenant cleanup after tenant deletion in tests --- .../server/controller/AbstractWebTest.java | 14 ++++++++++++-- .../housekeeper/HousekeeperServiceTest.java | 1 - .../src/test/resources/application-test.properties | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 63c974fa96..85edac9933 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -100,6 +100,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; @@ -273,7 +275,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { Tenant tenant = new Tenant(); tenant.setTitle(TEST_TENANT_NAME); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = createTenant(tenant); Assert.assertNotNull(savedTenant); tenantId = savedTenant.getId(); tenantProfileId = savedTenant.getTenantProfileId(); @@ -367,6 +369,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { throw new RuntimeException(e); } } + Awaitility.await("tenant cleanup finish").atMost(30, TimeUnit.SECONDS) + .until(() -> tsService.findLatest(TenantId.SYS_TENANT_ID, tenantId, "test").get().isEmpty()); } private List getAllTenants() throws Exception { @@ -419,7 +423,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle(TEST_DIFFERENT_TENANT_NAME); - savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + savedDifferentTenant = createTenant(tenant); differentTenantId = savedDifferentTenant.getId(); Assert.assertNotNull(savedDifferentTenant); User differentTenantAdmin = new User(); @@ -429,6 +433,12 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { savedDifferentTenantUser = createUserAndLogin(differentTenantAdmin, DIFFERENT_TENANT_ADMIN_PASSWORD); } + protected Tenant createTenant(Tenant tenant) throws Exception { + tenant = doPost("/api/tenant", tenant, Tenant.class); + tsService.save(TenantId.SYS_TENANT_ID, tenant.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker ts to later know when Housekeeper finishes tenant cleanup + return tenant; + } + protected void loginDifferentCustomer() throws Exception { if (savedDifferentCustomer != null) { login(savedDifferentCustomer.getEmail(), CUSTOMER_USER_PASSWORD); diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 1204e36184..71c11a4e95 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -99,7 +99,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @DaoSqlTest @TestPropertySource(properties = { - "queue.core.housekeeper.enabled=true", "transport.http.enabled=true", "queue.core.housekeeper.reprocessing-start-delay-sec=1", "queue.core.housekeeper.task-reprocessing-delay-sec=2", diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 5e4900fdc1..92959e4552 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -54,4 +54,3 @@ sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 server.log_controller_error_stack_trace=false -queue.core.housekeeper.enabled=false From 1ed95b24a60a9356279d66726d67afc1054b8b43 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 18 Mar 2024 18:02:06 +0200 Subject: [PATCH 24/72] Clean up api usage state data on deletion --- .../housekeeper/HousekeeperServiceTest.java | 22 ++++++++++++++----- .../dao/usagerecord/ApiUsageStateService.java | 2 -- .../dao/attributes/BaseAttributesService.java | 1 - .../server/dao/rule/BaseRuleChainService.java | 1 - .../server/dao/tenant/TenantServiceImpl.java | 3 --- .../usagerecord/ApiUsageStateServiceImpl.java | 19 ++++++++-------- 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 71c11a4e95..97c87df5d1 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -28,6 +28,7 @@ import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; @@ -48,7 +49,6 @@ import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -68,11 +68,13 @@ import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.service.housekeeper.processor.TelemetryDeletionTaskProcessor; @@ -127,6 +129,10 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private AlarmDao alarmDao; @Autowired private RelationService relationService; + @Autowired + private ApiUsageStateDao apiUsageStateDao; + @Autowired + private EntityServiceRegistry entityServiceRegistry; @SpyBean private TelemetryDeletionTaskProcessor telemetryDeletionTaskProcessor; @@ -208,6 +214,9 @@ public class HousekeeperServiceTest extends AbstractControllerTest { tenantId = differentTenantId; createRelatedData(tenantId); + createDifferentTenantCustomer(); + createRelatedData(differentTenantCustomerId); + loginDifferentTenant(); Device device = createDevice("oi324rujoi", "oi324rujoi"); createRelatedData(device.getId()); @@ -228,6 +237,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { UserId userId = savedDifferentTenantUser.getId(); createRelatedData(userId); + ApiUsageState tenantApiUsageState = apiUsageStateDao.findApiUsageStateByEntityId(differentTenantId); + loginSysAdmin(); deleteDifferentTenant(); @@ -238,6 +249,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { verifyNoRelatedData(ruleNode2Id); verifyNoRelatedData(ruleChainId); verifyNoRelatedData(userId); + verifyNoRelatedData(differentTenantCustomerId); + verifyNoRelatedData(tenantApiUsageState.getId()); verifyNoRelatedData(tenantId); }); } @@ -323,11 +336,12 @@ public class HousekeeperServiceTest extends AbstractControllerTest { for (HousekeeperTaskType taskType : expectedTaskTypes) { verify(housekeeperClient).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); } + assertThat(entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()).findEntity(tenantId, entityId)).isEmpty(); assertThat(getLatestTelemetry(entityId)).isNull(); assertThat(getTimeseriesHistory(entityId)).isEmpty(); for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { - assertThat(getAttribute(entityId, scope, scope + ATTRIBUTE_KEY)).isNull(); + assertThat(attributesService.findAll(tenantId, entityId, scope).get()).isEmpty(); } assertThat(getEvents(entityId)).isEmpty(); assertThat(alarmDao.findEntityAlarmRecordsByEntityId(tenantId, entityId)).isEmpty(); @@ -388,10 +402,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(HousekeeperServiceTest.TELEMETRY_KEY, 0, System.currentTimeMillis(), 10, "DESC"))).get(); } - private AttributeKvEntry getAttribute(EntityId entityId, String scope, String key) throws Exception { - return attributesService.find(tenantId, entityId, scope, key).get().orElse(null); - } - private List getEvents(EntityId entityId) { return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100)).getData() .stream().filter(event -> Optional.ofNullable(event.getBody()).map(body -> body.get("event")) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index 2ed877dabd..3d09247c7e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -31,8 +31,6 @@ public interface ApiUsageStateService extends EntityDaoService { ApiUsageState findApiUsageStateByEntityId(EntityId entityId); - void deleteApiUsageStateByTenantId(TenantId tenantId); - void deleteApiUsageStateByEntityId(EntityId entityId); ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 9359383071..1cace5b71c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -118,7 +118,6 @@ public class BaseAttributesService implements AttributesService { @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); - // todo: probably better to retrieve by scope in case there are too many attributes return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 18f3d58ba7..3c6489146c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -178,7 +178,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - cleanUpService.cleanUpRelatedData(tenantId, existingNode.getId()); // fixme: for sure? Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 181989a620..83e9ca8ed5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -54,7 +53,6 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service("TenantDaoService") @Slf4j -@RequiredArgsConstructor public class TenantServiceImpl extends AbstractCachedEntityService implements TenantService { private static final String DEFAULT_TENANT_REGION = "Global"; @@ -153,7 +151,6 @@ public class TenantServiceImpl extends AbstractCachedEntityService Date: Thu, 21 Mar 2024 12:43:34 +0200 Subject: [PATCH 25/72] Fix attributes removal by entity id --- .../server/common/data/AttributeScope.java | 11 +++++++++++ .../server/dao/attributes/BaseAttributesService.java | 2 +- .../dao/attributes/CachedAttributesService.java | 10 ++++++---- .../server/dao/sql/attributes/JpaAttributeDao.java | 10 +++++----- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java index 0e8335c7cb..3c58dd3b72 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java @@ -17,6 +17,10 @@ package org.thingsboard.server.common.data; import lombok.Getter; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + public enum AttributeScope { CLIENT_SCOPE(1), @@ -25,8 +29,15 @@ public enum AttributeScope { @Getter private final int id; + private static final Map values = Arrays.stream(values()) + .collect(Collectors.toMap(AttributeScope::getId, scope -> scope)); + AttributeScope(int id) { this.id = id; } + public static AttributeScope valueOf(int id) { + return values.get(id); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 4a63f4c04d..60856d361b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -164,7 +164,7 @@ public class BaseAttributesService implements AttributesService { @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { - List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); return deleted.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index ad3e0ecf3c..45e671a891 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; @@ -40,7 +41,6 @@ import org.thingsboard.server.dao.cache.CacheExecutorService; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.sql.JpaExecutorService; -import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -308,11 +308,13 @@ public class CachedAttributesService implements AttributesService { @Override public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { - List> result = attributesDao.removeAllByEntityId(tenantId, entityId); + List> result = attributesDao.removeAllByEntityId(tenantId, entityId); result.forEach(deleted -> { - String scope = deleted.getKey(); + AttributeScope scope = deleted.getKey(); String key = deleted.getValue(); - cache.evict(new AttributeCacheKey(scope, entityId, key)); + if (scope != null && key != null) { + cache.evict(new AttributeCacheKey(scope, entityId, key)); + } }); return result.size(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index a227d1ed50..3bf8e43c70 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; @@ -209,10 +209,10 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl @Transactional @Override public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { - return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_type = ? and entity_id = ? " + - "RETURNING attribute_type, attribute_key", entityId.getEntityType().name(), entityId.getId()).stream() - .map(row -> Pair.of(row.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN), - (String) row.get(ModelConstants.ATTRIBUTE_KEY_COLUMN))) + return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_id = ? " + + "RETURNING attribute_type, attribute_key", entityId.getId()).stream() + .map(row -> Pair.of(AttributeScope.valueOf((Integer) row.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN)), + keyDictionaryDao.getKey((Integer) row.get(ModelConstants.ATTRIBUTE_KEY_COLUMN)))) .collect(Collectors.toList()); } From e5d48921649d542417eff83e0f11685daec3e37b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 27 Mar 2024 13:25:42 +0200 Subject: [PATCH 26/72] Separate latest and ts history deletion tasks for each key --- .../HousekeeperReprocessingService.java | 2 - .../housekeeper/HousekeeperService.java | 2 - .../AlarmsDeletionTaskProcessor.java | 2 +- .../AlarmsUnassignTaskProcessor.java | 2 +- .../AttributesDeletionTaskProcessor.java | 2 +- .../EntitiesDeletionTaskProcessor.java | 2 +- .../EventsDeletionTaskProcessor.java | 2 +- .../processor/HousekeeperTaskProcessor.java | 11 +++-- .../LatestTsDeletionTaskProcessor.java | 45 +++++++++++++++++ .../TelemetryDeletionTaskProcessor.java | 23 ++++++--- .../TsHistoryDeletionTaskProcessor.java | 48 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 2 - .../housekeeper/HousekeeperServiceTest.java | 28 ++++++----- .../data/housekeeper/HousekeeperTask.java | 11 +++++ .../data/housekeeper/HousekeeperTaskType.java | 2 + .../LatestTsDeletionHousekeeperTask.java | 37 ++++++++++++++ .../TsHistoryDeletionHousekeeperTask.java | 37 ++++++++++++++ 17 files changed, 224 insertions(+), 34 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 8b1dece58e..7410ec2903 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -49,7 +48,6 @@ import java.util.concurrent.atomic.AtomicInteger; @TbCoreComponent @Service @Slf4j -@ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) public class HousekeeperReprocessingService { private final HousekeeperService housekeeperService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index fa59eae4c9..1a8d07c9da 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -51,7 +50,6 @@ import java.util.stream.Collectors; @TbCoreComponent @Service @Slf4j -@ConditionalOnProperty(name = "queue.core.housekeeper.enabled", havingValue = "true", matchIfMissing = true) public class HousekeeperService { private final Map> taskProcessors; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index 2d7cf690d9..eea0651873 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -26,7 +26,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; @Component @RequiredArgsConstructor @Slf4j -public class AlarmsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor { private final AlarmService alarmService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java index 9a8517903f..3ba6ecde52 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -29,7 +29,7 @@ import java.util.List; @Component @RequiredArgsConstructor @Slf4j -public class AlarmsUnassignTaskProcessor implements HousekeeperTaskProcessor { +public class AlarmsUnassignTaskProcessor extends HousekeeperTaskProcessor { private final TbAlarmService alarmService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java index c8ff5306a7..5514921e2e 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor @Slf4j -public class AttributesDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class AttributesDeletionTaskProcessor extends HousekeeperTaskProcessor { private final AttributesService attributesService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index a16a5f09bd..95d45a49be 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.entity.EntityServiceRegistry; @Component @RequiredArgsConstructor -public class EntitiesDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { private final EntityServiceRegistry entityServiceRegistry; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java index 551c8ce7e0..bf9c528eb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; @Component @RequiredArgsConstructor -public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class EventsDeletionTaskProcessor extends HousekeeperTaskProcessor { private final EventService eventService; diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java index 79e87fcc91..f193b154db 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -15,13 +15,18 @@ */ package org.thingsboard.server.service.housekeeper.processor; +import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; -public interface HousekeeperTaskProcessor { +public abstract class HousekeeperTaskProcessor { - void process(T task) throws Exception; + @Autowired + protected HousekeeperClient housekeeperClient; - HousekeeperTaskType getTaskType(); + public abstract void process(T task) throws Exception; + + public abstract HousekeeperTaskType getTaskType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java new file mode 100644 index 0000000000..1f4b6b57ae --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class LatestTsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(LatestTsDeletionHousekeeperTask task) throws Exception { + timeseriesService.removeLatest(task.getTenantId(), task.getEntityId(), List.of(task.getKey())).get(); + log.debug("[{}][{}][{}] Deleted latest telemetry for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_LATEST_TS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java index c7f37ed091..5271e77c87 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -18,10 +18,12 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.List; @@ -29,18 +31,25 @@ import java.util.List; @Component @RequiredArgsConstructor @Slf4j -public class TelemetryDeletionTaskProcessor implements HousekeeperTaskProcessor { +public class TelemetryDeletionTaskProcessor extends HousekeeperTaskProcessor { private final TimeseriesService timeseriesService; @Override public void process(HousekeeperTask task) throws Exception { - List keys = timeseriesService.findAllKeysByEntityIds(task.getTenantId(), List.of(task.getEntityId())); + TenantId tenantId = task.getTenantId(); + EntityId entityId = task.getEntityId(); + List keys = timeseriesService.findAllKeysByEntityIds(tenantId, List.of(entityId)); + for (String key : keys) { - DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false, true); - timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); + var latestTsDeletionTask = new LatestTsDeletionHousekeeperTask(tenantId, entityId, key); + housekeeperClient.submitTask(latestTsDeletionTask); + + var tsHistoryDeletionTask = new TsHistoryDeletionHousekeeperTask(tenantId, entityId, key); + housekeeperClient.submitTask(tsHistoryDeletionTask); } - log.debug("[{}][{}][{}] Deleted {} telemetry keys", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), keys.size()); + + log.trace("[{}][{}][{}] Submitted latest and ts history deletion tasks for {} keys", tenantId, entityId.getEntityType(), entityId, keys.size()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java new file mode 100644 index 0000000000..cee99ae93e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TsHistoryDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(TsHistoryDeletionHousekeeperTask task) throws Exception { + DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(task.getKey(), 0, System.currentTimeMillis(), false, false); + timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); + log.debug("[{}][{}][{}] Deleted timeseries history for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_TS_HISTORY; + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1645bdc84d..99b173399e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1624,8 +1624,6 @@ queue: # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" housekeeper: - # Enabled/disable Housekeeping service, which does cleanup of telemetry, attributes, events, etc. - enabled: "${TB_HOUSEKEEPER_ENABLED:true}" # Topic name for Housekeeper tasks topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" # Topic name for Housekeeper tasks to be reprocessed diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 97c87df5d1..44891c2b7f 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -29,6 +29,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; @@ -78,6 +79,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.service.housekeeper.processor.TelemetryDeletionTaskProcessor; +import org.thingsboard.server.service.housekeeper.processor.TsHistoryDeletionTaskProcessor; import java.util.Arrays; import java.util.Collections; @@ -134,7 +136,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Autowired private EntityServiceRegistry entityServiceRegistry; @SpyBean - private TelemetryDeletionTaskProcessor telemetryDeletionTaskProcessor; + private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; private TenantId tenantId; @@ -258,7 +260,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenTaskProcessingFails_thenReprocessUntilSuccessful() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); - doThrow(error).when(telemetryDeletionTaskProcessor).process(any()); + doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); Device device = createDevice("vep9ruv32", "vep9ruv32"); createRelatedData(device.getId()); @@ -267,17 +269,17 @@ public class HousekeeperServiceTest extends AbstractControllerTest { int attempts = 3; await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, 0); + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, 0); for (int i = 1; i <= attempts; i++) { int attempt = i; - verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, + verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, attempt); + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, attempt); } }); assertThat(getTimeseriesHistory(device.getId())).isNotEmpty(); - doCallRealMethod().when(telemetryDeletionTaskProcessor).process(any()); // fixing the code + doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); // fixing the code await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(getTimeseriesHistory(device.getId())).isEmpty(); }); @@ -286,7 +288,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenReprocessingAttemptsExceeded_thenReprocessOnNextStartUp() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); - doThrow(error).when(telemetryDeletionTaskProcessor).process(any()); + doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); Device device = createDevice("woeifjiowejf", "woeifjiowejf"); createRelatedData(device.getId()); @@ -296,18 +298,18 @@ public class HousekeeperServiceTest extends AbstractControllerTest { int maxAttempts = 5; await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { for (int i = 1; i <= maxAttempts; i++) { - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, i); + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, i); } }); Mockito.clearInvocations(housekeeperService); - doCallRealMethod().when(telemetryDeletionTaskProcessor).process(any()); + doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); TimeUnit.SECONDS.sleep(2); - verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, null))); + verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, null))); housekeeperReprocessingService.cycle.set(0); // imitating start-up await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TELEMETRY, 6); + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, 6); }); } @@ -325,7 +327,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private void createRelatedData(EntityId entityId) throws Exception { createTelemetry(entityId); - for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { + for (AttributeScope scope : List.of(AttributeScope.SERVER_SCOPE, AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE)) { createAttribute(entityId, scope, scope + ATTRIBUTE_KEY); } createEvent(entityId); @@ -351,7 +353,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { assertThat(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); } - private void createAttribute(EntityId entityId, String scope, String key) throws Exception { + private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception { attributesService.save(tenantId, entityId, scope, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, KV_VALUE))).get(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index bc41f553ad..08862d26be 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -34,6 +34,8 @@ import java.io.Serializable; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "taskType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = HousekeeperTask.class) @JsonSubTypes({ + @Type(name = "DELETE_TS_HISTORY", value = TsHistoryDeletionHousekeeperTask.class), + @Type(name = "DELETE_LATEST_TS", value = LatestTsDeletionHousekeeperTask.class), @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) }) @@ -57,10 +59,19 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); } + /** + * Task for telemetry deletion, processed by TelemetryDeletionTaskProcessor. + * The processor will submit separate tasks for each key for latest and ts history deletion, + * processed by LatestTsDeletionTaskProcessor and TsHistoryDeletionTaskProcessor accordingly. + * */ public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY); } + /** + * Task for events deletion, processed by EventsDeletionTaskProcessor. + * All types of events will be dropped for this entity. + * */ public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index eb5833b06f..572e85f2cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -25,6 +25,8 @@ public enum HousekeeperTaskType { DELETE_ENTITIES("entities deletion"), DELETE_ATTRIBUTES("attributes deletion"), DELETE_TELEMETRY("telemetry deletion"), + DELETE_LATEST_TS("latest telemetry deletion"), + DELETE_TS_HISTORY("timeseries history deletion"), DELETE_EVENTS("events deletion"), DELETE_ALARMS("alarms deletion"), UNASSIGN_ALARMS("alarms unassigning"); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java new file mode 100644 index 0000000000..59e815bbfa --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { + + private String key; + + public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_LATEST_TS); + this.key = key; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java new file mode 100644 index 0000000000..1656626e6b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { + + private String key; + + public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_TS_HISTORY); + this.key = key; + } + +} From f621d59be323a9c9e1e5fc8a331ba52f0b8ce887 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Thu, 4 Apr 2024 10:45:41 +0300 Subject: [PATCH 27/72] Added logic when ping requests for gateway device will be also activity events for devices, connected through the gateway --- .../transport/mqtt/MqttTransportHandler.java | 16 +++- .../AbstractGatewaySessionHandler.java | 36 +++++++- .../mqtt/session/GatewaySessionHandler.java | 6 +- .../session/SparkplugNodeSessionHandler.java | 4 +- .../session/GatewaySessionHandlerTest.java | 83 +++++++++++++++++++ 5 files changed, 132 insertions(+), 13 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 8360111a54..2ecea63e42 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -45,6 +45,7 @@ import io.netty.util.concurrent.GenericFutureListener; import lombok.extern.slf4j.Slf4j; import org.eclipse.leshan.core.ResponseCode; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -64,7 +65,6 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.adaptor.AdaptorException; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; @@ -323,6 +323,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (checkConnected(ctx, msg)) { ctx.writeAndFlush(new MqttMessage(new MqttFixedHeader(PINGRESP, false, AT_MOST_ONCE, false, 0))); transportService.recordActivity(deviceSessionCtx.getSessionInfo()); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onGatewayPing(); + } } break; case DISCONNECT: @@ -1082,10 +1085,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement if (infoNode != null) { JsonNode gatewayNode = infoNode.get("gateway"); if (gatewayNode != null && gatewayNode.asBoolean()) { - gatewaySessionHandler = new GatewaySessionHandler(deviceSessionCtx, sessionId); + boolean overwriteDevicesActivity = false; if (infoNode.has(DefaultTransportService.OVERWRITE_ACTIVITY_TIME) && infoNode.get(DefaultTransportService.OVERWRITE_ACTIVITY_TIME).isBoolean()) { - sessionMetaData.setOverwriteActivityTime(infoNode.get(DefaultTransportService.OVERWRITE_ACTIVITY_TIME).asBoolean()); + overwriteDevicesActivity = infoNode.get(DefaultTransportService.OVERWRITE_ACTIVITY_TIME).asBoolean(); + sessionMetaData.setOverwriteActivityTime(overwriteDevicesActivity); } + gatewaySessionHandler = new GatewaySessionHandler(deviceSessionCtx, sessionId, overwriteDevicesActivity); } } } catch (IOException e) { @@ -1099,7 +1104,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement SparkplugTopic sparkplugTopicNode = validatedSparkplugTopicConnectedNode(connectMessage); if (sparkplugTopicNode != null) { SparkplugBProto.Payload sparkplugBProtoNode = SparkplugBProto.Payload.parseFrom(connectMessage.payload().willMessageInBytes()); - sparkplugSessionHandler = new SparkplugNodeSessionHandler(this, deviceSessionCtx, sessionId, sparkplugTopicNode); + sparkplugSessionHandler = new SparkplugNodeSessionHandler(this, deviceSessionCtx, sessionId, true, sparkplugTopicNode); sparkplugSessionHandler.onAttributesTelemetryProto(0, sparkplugBProtoNode, sparkplugTopicNode); sessionMetaData.setOverwriteActivityTime(true); } else { @@ -1354,6 +1359,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement @Override public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { deviceSessionCtx.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); + if (gatewaySessionHandler != null) { + gatewaySessionHandler.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); + } } @Override diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 1cb391b1f4..dc6ba367ab 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -32,17 +32,21 @@ import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttPublishMessage; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import org.springframework.util.ConcurrentReferenceHashMap; +import org.thingsboard.server.common.adaptor.AdaptorException; +import org.thingsboard.server.common.adaptor.JsonConverter; +import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.adaptor.AdaptorException; -import org.thingsboard.server.common.adaptor.JsonConverter; -import org.thingsboard.server.common.adaptor.ProtoConverter; import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGatewayResponse; import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.gen.transport.TransportApiProtos; @@ -64,6 +68,7 @@ import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -89,6 +94,8 @@ public abstract class AbstractGatewaySessionHandler createWeakMap() { @@ -164,6 +176,12 @@ public abstract class AbstractGatewaySessionHandler transportService.recordActivity(deviceSessionCtx.getSessionInfo())); + } + } + public void onDevicesDisconnect() { devices.forEach(this::deregisterSession); } @@ -221,6 +239,16 @@ public abstract class AbstractGatewaySessionHandler deviceProfileOpt) { + log.trace("[{}][{}] onDeviceUpdate: [{}]", gateway.getTenantId(), gateway.getDeviceId(), device); + if (device.getAdditionalInfo().has(GATEWAY_PROPERTY) + && device.getAdditionalInfo().get(GATEWAY_PROPERTY).asBoolean() + && device.getAdditionalInfo().has(OVERWRITE_ACTIVITY_TIME) + && device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).isBoolean()) { + overwriteDevicesActivity = device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).asBoolean(); + } + } + ListenableFuture onDeviceConnect(String deviceName, String deviceType) { T result = devices.get(deviceName); if (result == null) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java index 9ccc88aa82..9aa228afff 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandler.java @@ -25,10 +25,10 @@ import java.util.UUID; /** * Created by nickAS21 on 26.12.22 */ -public class GatewaySessionHandler extends AbstractGatewaySessionHandler { +public class GatewaySessionHandler extends AbstractGatewaySessionHandler { - public GatewaySessionHandler(DeviceSessionCtx deviceSessionCtx, UUID sessionId) { - super(deviceSessionCtx, sessionId); + public GatewaySessionHandler(DeviceSessionCtx deviceSessionCtx, UUID sessionId, boolean overwriteDevicesActivity) { + super(deviceSessionCtx, sessionId, overwriteDevicesActivity); } public void onDeviceConnect(MqttPublishMessage mqttMsg) throws AdaptorException { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java index d4f2364d79..67dda4681a 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/SparkplugNodeSessionHandler.java @@ -73,8 +73,8 @@ public class SparkplugNodeSessionHandler extends AbstractGatewaySessionHandler(); diff --git a/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java b/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java index bd1ddd2d54..9b52d57dc2 100644 --- a/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java +++ b/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java @@ -15,15 +15,98 @@ */ package org.thingsboard.server.transport.mqtt.session; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; import org.springframework.util.ConcurrentReferenceHashMap; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; +import org.thingsboard.server.transport.mqtt.MqttTransportContext; + +import java.lang.reflect.Field; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; public class GatewaySessionHandlerTest { + @Mock + private TransportService transportService; + + @Mock + private DeviceSessionCtx deviceSessionCtx; + + @Mock + private MqttTransportContext transportContext; + + private GatewaySessionHandler handler; + + @BeforeEach + public void setup() { + MockitoAnnotations.openMocks(this); + when(deviceSessionCtx.getSessionId()).thenReturn(UUID.randomUUID()); + doNothing().when(transportService).recordActivity(any()); + when(transportContext.getTransportService()).thenReturn(transportService); + when(deviceSessionCtx.getContext()).thenReturn(transportContext); + handler = new GatewaySessionHandler(deviceSessionCtx, UUID.randomUUID(), true); + when(handler.getNodeId()).thenReturn("nodeId"); + } + + @Test + public void shouldRecordActivityWhenOnGatewayPing() throws Exception { + // Given + ConcurrentHashMap devices = new ConcurrentHashMap<>(); + TransportDeviceInfo deviceInfo = new TransportDeviceInfo(); + deviceInfo.setDeviceId(new DeviceId(UUID.randomUUID())); + deviceInfo.setTenantId(new TenantId(UUID.randomUUID())); + deviceInfo.setCustomerId(new CustomerId(UUID.randomUUID())); + deviceInfo.setDeviceName("device1"); + deviceInfo.setDeviceType("default"); + deviceInfo.setDeviceProfileId(new DeviceProfileId(UUID.randomUUID())); + deviceInfo.setAdditionalInfo("{\"gateway\": true, \"overwriteDeviceActivity\": true}"); + when(deviceSessionCtx.getDeviceInfo()).thenReturn(deviceInfo); + GatewayDeviceSessionContext gatewayDeviceSessionContext = new GatewayDeviceSessionContext(handler, deviceInfo, null, null, transportService); + devices.put("device1", gatewayDeviceSessionContext); + when(handler.getNodeId()).thenReturn("nodeId"); + Field devicesField = AbstractGatewaySessionHandler.class.getDeclaredField("devices"); + devicesField.setAccessible(true); + devicesField.set(handler, devices); + + // When + handler.onGatewayPing(); + + // Then + verify(transportService).recordActivity(gatewayDeviceSessionContext.getSessionInfo()); + } + + @Test + public void shouldNotRecordActivityWhenNoDevicesOnGatewayPing() throws Exception { + // Given + ConcurrentHashMap devices = new ConcurrentHashMap<>(); + Field devicesField = AbstractGatewaySessionHandler.class.getDeclaredField("devices"); + devicesField.setAccessible(true); + devicesField.set(handler, devices); + + // When + handler.onGatewayPing(); + + // Then + verify(transportService, never()).recordActivity(any()); + } + @Test public void givenGatewaySessionHandler_WhenCreateWeakMap_thenConcurrentReferenceHashMapClass() { GatewaySessionHandler gsh = mock(GatewaySessionHandler.class); From bead94d8327f068bfd1cf0988d310631383194ab Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 8 Apr 2024 11:45:26 +0300 Subject: [PATCH 28/72] Changes due to comments --- .../mqtt/session/AbstractGatewaySessionHandler.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index dc6ba367ab..3be1d632be 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -16,6 +16,7 @@ package org.thingsboard.server.transport.mqtt.session; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -241,11 +242,12 @@ public abstract class AbstractGatewaySessionHandler deviceProfileOpt) { log.trace("[{}][{}] onDeviceUpdate: [{}]", gateway.getTenantId(), gateway.getDeviceId(), device); - if (device.getAdditionalInfo().has(GATEWAY_PROPERTY) - && device.getAdditionalInfo().get(GATEWAY_PROPERTY).asBoolean() - && device.getAdditionalInfo().has(OVERWRITE_ACTIVITY_TIME) - && device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).isBoolean()) { - overwriteDevicesActivity = device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).asBoolean(); + JsonNode deviceAdditionalInfo = device.getAdditionalInfo(); + if (deviceAdditionalInfo.has(GATEWAY_PROPERTY) + && deviceAdditionalInfo.get(GATEWAY_PROPERTY).asBoolean() + && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME) + && deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).isBoolean()) { + overwriteDevicesActivity = deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).asBoolean(); } } From a6143f3c8abe1db72c80f373bca069511c1b353e Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 8 Apr 2024 13:05:58 +0300 Subject: [PATCH 29/72] Improvement for update of overwrite activity time --- .../mqtt/session/AbstractGatewaySessionHandler.java | 5 +---- .../transport/service/DefaultTransportService.java | 9 ++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 3be1d632be..b881291756 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -243,10 +243,7 @@ public abstract class AbstractGatewaySessionHandler deviceProfileOpt) { log.trace("[{}][{}] onDeviceUpdate: [{}]", gateway.getTenantId(), gateway.getDeviceId(), device); JsonNode deviceAdditionalInfo = device.getAdditionalInfo(); - if (deviceAdditionalInfo.has(GATEWAY_PROPERTY) - && deviceAdditionalInfo.get(GATEWAY_PROPERTY).asBoolean() - && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME) - && deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).isBoolean()) { + if (deviceAdditionalInfo.has(GATEWAY_PROPERTY) && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME)) { overwriteDevicesActivity = deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).asBoolean(); } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 4aa989ed25..2f822ccb1a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -1047,11 +1048,9 @@ public class DefaultTransportService extends TransportActivityManager implements .setDeviceProfileIdLSB(deviceProfileIdLSB) .setDeviceName(device.getName()) .setDeviceType(device.getType()).build(); - if (device.getAdditionalInfo().has("gateway") - && device.getAdditionalInfo().get("gateway").asBoolean() - && device.getAdditionalInfo().has(OVERWRITE_ACTIVITY_TIME) - && device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).isBoolean()) { - md.setOverwriteActivityTime(device.getAdditionalInfo().get(OVERWRITE_ACTIVITY_TIME).asBoolean()); + JsonNode deviceAdditionalInfo = device.getAdditionalInfo(); + if (deviceAdditionalInfo.has("gateway") && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME)) { + md.setOverwriteActivityTime(deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).asBoolean()); } md.setSessionInfo(newSessionInfo); transportCallbackExecutor.submit(() -> md.getListener().onDeviceUpdate(newSessionInfo, device, Optional.ofNullable(newDeviceProfile))); From 49a76cad5a4e0641634d047db30c8cce3d532742 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 11 Apr 2024 11:53:32 +0300 Subject: [PATCH 30/72] Housekeeper: disabled task types; refactoring --- .../HousekeeperReprocessingService.java | 169 +++++------------- .../housekeeper/HousekeeperService.java | 120 ++++++------- .../stats/HousekeeperStatsService.java | 2 +- .../queue/consumer/QueueConsumerManager.java | 109 +++++++++++ .../src/main/resources/thingsboard.yml | 12 +- .../housekeeper/HousekeeperServiceTest.java | 19 +- .../AlarmsUnassignHousekeeperTask.java | 2 + .../EntitiesDeletionHousekeeperTask.java | 2 + .../LatestTsDeletionHousekeeperTask.java | 2 + .../TsHistoryDeletionHousekeeperTask.java | 2 + .../housekeeper/DefaultHousekeeperClient.java | 14 +- .../queue/housekeeper/HousekeeperConfig.java | 40 +++++ .../common/util/ThingsBoardThreadFactory.java | 5 + 13 files changed, 289 insertions(+), 209 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index 7410ec2903..ba762facb1 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -17,23 +17,22 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.consumer.QueueConsumerManager; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.LinkedHashSet; import java.util.List; @@ -41,8 +40,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @TbCoreComponent @@ -50,160 +47,74 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public class HousekeeperReprocessingService { + private final HousekeeperConfig config; private final HousekeeperService housekeeperService; - private final PartitionService partitionService; - private final TbCoreQueueFactory queueFactory; - private TbQueueProducer> producer; - private TopicPartitionInfo submitTpi; - - @Value("${queue.core.housekeeper.reprocessing-start-delay-sec:300}") - private int startDelay; - @Value("${queue.core.housekeeper.task-reprocessing-delay-sec:3600}") - private int reprocessingDelay; - @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") - private int maxReprocessingAttempts; - @Value("${queue.core.housekeeper.poll-interval-ms:500}") - private int pollInterval; + private final QueueConsumerManager> consumer; + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-scheduler")); protected AtomicInteger cycle = new AtomicInteger(); - private boolean stopped; - public HousekeeperReprocessingService(@Lazy HousekeeperService housekeeperService, - PartitionService partitionService, + public HousekeeperReprocessingService(HousekeeperConfig config, + @Lazy HousekeeperService housekeeperService, TbCoreQueueFactory queueFactory) { + this.config = config; this.housekeeperService = housekeeperService; - this.partitionService = partitionService; - this.queueFactory = queueFactory; - } - - @PostConstruct - private void init() { - producer = queueFactory.createHousekeeperReprocessingMsgProducer(); - submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); - - scheduler.scheduleWithFixedDelay(() -> { - try { - cycle.incrementAndGet(); - startReprocessing(); - } catch (Throwable e) { - log.error("Unexpected error during reprocessing", e); - } - }, startDelay, reprocessingDelay, TimeUnit.SECONDS); + this.consumer = QueueConsumerManager.>builder() + .name("Housekeeper reprocessing") + .msgPackProcessor(this::processMsgs) + .pollInterval(config.getPollInterval()) + .consumerCreator(queueFactory::createHousekeeperReprocessingMsgConsumer) + .consumerExecutor(consumerExecutor) + .build(); + this.producer = queueFactory.createHousekeeperReprocessingMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); } - public void startReprocessing() { - if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { - return; - } - - var consumer = queueFactory.createHousekeeperReprocessingMsgConsumer(); - consumer.subscribe(); - consumerExecutor.submit(() -> { - log.info("Starting Housekeeper tasks reprocessing"); - long startTs = System.currentTimeMillis(); - while (!stopped) { - try { - List> msgs = consumer.poll(pollInterval); - if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { - // it's not time yet to process the message - if (!consumer.isCommitSupported()) { - // resubmitting consumed messages if committing is not supported (for in-memory queue) - for (TbProtoQueueMsg msg : msgs) { - submit(msg.getKey(), msg.getValue()); - } - } - break; - } - - for (TbProtoQueueMsg msg : msgs) { - log.trace("Reprocessing task: {}", msg); - try { - reprocessTask(msg.getValue()); - } catch (InterruptedException e) { - return; - } catch (Throwable e) { - log.error("Unexpected error during message reprocessing [{}]", msg, e); - submitForReprocessing(msg.getValue(), e); - } - } - consumer.commit(); - } catch (Throwable t) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", t); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException interruptedException) { - log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); - } - } - } - } - consumer.unsubscribe(); - log.info("Stopped Housekeeper tasks reprocessing"); - }); + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); // Kafka topic for tasks reprocessing has only 1 partition, so only one TB Core will reprocess tasks + consumer.launch(); } - private void reprocessTask(ToHousekeeperServiceMsg msg) throws Exception { - int attempt = msg.getTask().getAttempt(); - if (attempt > maxReprocessingAttempts) { - if (cycle.get() == 1) { // only reprocessing tasks with exceeded failures on first cycle (after start-up) - log.info("Trying to reprocess task with {} failed attempts: {}", attempt, msg); - } else { - // resubmitting msg to be processed on the next service start - msg = msg.toBuilder() - .setTask(msg.getTask().toBuilder() - .setTs(getReprocessingTs()) - .build()) - .build(); - submit(UUID.randomUUID(), msg); + private void processMsgs(List> msgs, TbQueueConsumer> consumer) throws Exception { + for (TbProtoQueueMsg msg : msgs) { + log.trace("Reprocessing task: {}", msg); + try { + housekeeperService.processTask(msg.getValue()); + } catch (InterruptedException e) { return; + } catch (Throwable e) { + log.error("Unexpected error during message reprocessing [{}]", msg, e); + submitForReprocessing(msg.getValue(), e); } } + consumer.commit(); - housekeeperService.processTask(msg); + Thread.sleep(config.getTaskReprocessingDelay()); } public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { HousekeeperTaskProto task = msg.getTask(); - - int attempt = task.getAttempt() + 1; Set errors = new LinkedHashSet<>(task.getErrorsList()); errors.add(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)); msg = msg.toBuilder() .setTask(task.toBuilder() - .setAttempt(attempt) + .setAttempt(task.getAttempt() + 1) .clearErrors().addAllErrors(errors) - .setTs(getReprocessingTs()) .build()) .build(); log.trace("Submitting for reprocessing: {}", msg); - submit(UUID.randomUUID(), msg); // reprocessing topic has single partition, so we don't care about the msg key - - if (task.getAttempt() >= maxReprocessingAttempts) { - log.warn("Failed to process task in {} attempts: {}", task.getAttempt(), msg); - } - } - - private void submit(UUID key, ToHousekeeperServiceMsg msg) { - producer.send(submitTpi, new TbProtoQueueMsg<>(key, msg), null); - } - - private long getReprocessingTs() { - return System.currentTimeMillis() + TimeUnit.SECONDS.toMillis((long) (reprocessingDelay * 0.8)); // *0.8 so that msgs submitted just after finishing reprocessing are processed on the next cycle + producer.send(submitTpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @PreDestroy - private void stop() throws Exception { - stopped = true; - scheduler.shutdownNow(); - consumerExecutor.shutdown(); - if (!consumerExecutor.awaitTermination(10, TimeUnit.SECONDS)) { - consumerExecutor.shutdownNow(); - } + private void stop() { + consumer.stop(); + consumerExecutor.shutdownNow(); log.info("Stopped Housekeeper reprocessing service"); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 1a8d07c9da..89953676c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.housekeeper; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -29,11 +28,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; +import org.thingsboard.server.service.queue.consumer.QueueConsumerManager; import javax.annotation.PreDestroy; import java.util.List; @@ -54,90 +55,79 @@ public class HousekeeperService { private final Map> taskProcessors; + private final HousekeeperConfig config; private final HousekeeperReprocessingService reprocessingService; private final Optional statsService; private final NotificationRuleProcessor notificationRuleProcessor; - private final TbQueueConsumer> consumer; + private final QueueConsumerManager> consumer; private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); - private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); + private final ExecutorService taskExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); - @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") - private int taskProcessingTimeout; - @Value("${queue.core.housekeeper.poll-interval-ms:500}") - private int pollInterval; - - private boolean stopped; - - public HousekeeperService(HousekeeperReprocessingService reprocessingService, + public HousekeeperService(HousekeeperConfig config, + HousekeeperReprocessingService reprocessingService, TbCoreQueueFactory queueFactory, Optional statsService, NotificationRuleProcessor notificationRuleProcessor, @Lazy List> taskProcessors) { + this.config = config; this.reprocessingService = reprocessingService; this.statsService = statsService; this.notificationRuleProcessor = notificationRuleProcessor; - this.consumer = queueFactory.createHousekeeperMsgConsumer(); + this.consumer = QueueConsumerManager.>builder() + .name("Housekeeper") + .msgPackProcessor(this::processMsgs) + .pollInterval(config.getPollInterval()) + .consumerCreator(queueFactory::createHousekeeperMsgConsumer) + .consumerExecutor(consumerExecutor) + .build(); this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); } @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) public void afterStartUp() { consumer.subscribe(); - consumerExecutor.submit(() -> { - while (!stopped && !consumer.isStopped()) { - try { - List> msgs = consumer.poll(pollInterval); - if (msgs.isEmpty()) { - continue; - } + consumer.launch(); + } - for (TbProtoQueueMsg msg : msgs) { - log.trace("Processing task: {}", msg); - try { - processTask(msg.getValue()); - } catch (InterruptedException e) { - return; - } catch (Throwable e) { - log.error("Unexpected error during message processing [{}]", msg, e); - reprocessingService.submitForReprocessing(msg.getValue(), e); - } - } - consumer.commit(); - } catch (Throwable t) { - if (!consumer.isStopped()) { - log.warn("Failed to process messages from queue", t); - try { - Thread.sleep(pollInterval); - } catch (InterruptedException interruptedException) { - log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); - } - } - } + private void processMsgs(List> msgs, TbQueueConsumer> consumer) { + for (TbProtoQueueMsg msg : msgs) { + log.trace("Processing task: {} attempt={}", msg, msg.getValue().getTask().getAttempt()); + try { + processTask(msg.getValue()); + } catch (InterruptedException e) { + return; + } catch (Throwable e) { + log.error("Unexpected error during message processing [{}]", msg, e); + reprocessingService.submitForReprocessing(msg.getValue(), e); } - }); - log.info("Started Housekeeper service"); + } + consumer.commit(); } @SuppressWarnings("unchecked") protected void processTask(ToHousekeeperServiceMsg msg) throws Exception { HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); - HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(task.getTaskType()); + HousekeeperTaskType taskType = task.getTaskType(); + if (config.getDisabledTaskTypes().contains(taskType)) { + log.debug("Task type {} is disabled, ignoring {}", taskType, task); + return; + } + HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(taskType); if (taskProcessor == null) { - throw new IllegalArgumentException("Unsupported task type " + task.getTaskType()); + throw new IllegalArgumentException("Unsupported task type " + taskType); } if (log.isDebugEnabled()) { log.debug("[{}] {} {}", task.getTenantId(), isNew(msg.getTask()) ? "Processing" : "Reprocessing", task.getDescription()); } try { - Future future = executor.submit(() -> { + Future future = taskExecutor.submit(() -> { taskProcessor.process((T) task); return null; }); - future.get(taskProcessingTimeout, TimeUnit.MILLISECONDS); - - statsService.ifPresent(statsService -> statsService.reportProcessed(task.getTaskType(), msg)); + future.get(config.getTaskProcessingTimeout(), TimeUnit.MILLISECONDS); + statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg)); } catch (InterruptedException e) { throw e; } catch (Throwable e) { @@ -145,19 +135,23 @@ public class HousekeeperService { if (e instanceof ExecutionException) { error = e.getCause(); } else if (e instanceof TimeoutException) { - error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds"); + error = new TimeoutException("Timeout after " + config.getTaskProcessingTimeout() + " seconds"); } log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - task.getTaskType(), msg.getTask().getAttempt(), task, error); - reprocessingService.submitForReprocessing(msg, error); - - statsService.ifPresent(statsService -> statsService.reportFailure(task.getTaskType(), msg)); - notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() - .task(task) - .error(error) - .attempt(msg.getTask().getAttempt()) - .build()); + taskType, msg.getTask().getAttempt(), task, error); + + if (msg.getTask().getAttempt() < config.getMaxReprocessingAttempts()) { + reprocessingService.submitForReprocessing(msg, error); + } else { + log.error("Failed to process task in {} attempts: {}", msg.getTask().getAttempt(), msg); + notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() + .task(task) + .error(error) + .attempt(msg.getTask().getAttempt()) + .build()); + } + statsService.ifPresent(statsService -> statsService.reportFailure(taskType, msg)); } } @@ -167,12 +161,8 @@ public class HousekeeperService { @PreDestroy private void stop() throws Exception { - stopped = true; - consumer.unsubscribe(); - consumerExecutor.shutdown(); - if (!consumerExecutor.awaitTermination(10, TimeUnit.SECONDS)) { - consumerExecutor.shutdownNow(); - } + consumer.stop(); + consumerExecutor.shutdownNow(); log.info("Stopped Housekeeper service"); } diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 400bc36479..4944cf9f43 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -105,7 +105,7 @@ public class HousekeeperStatsService { } private StatsCounter register(String statsName, StatsFactory statsFactory) { - StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, Map.of("taskType", taskType.name())); + StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, "taskType", taskType.name()); counters.add(counter); return counter; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java new file mode 100644 index 0000000000..e7b3a2f2b7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.consumer; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; + +@Slf4j +public class QueueConsumerManager { + + private final String name; + private final MsgPackProcessor msgPackProcessor; + private final long pollInterval; + private final ExecutorService consumerExecutor; + private final String threadPrefix; + + @Getter + private final TbQueueConsumer consumer; + private volatile boolean stopped; + + @Builder + public QueueConsumerManager(String name, MsgPackProcessor msgPackProcessor, + long pollInterval, Supplier> consumerCreator, + ExecutorService consumerExecutor, String threadPrefix) { + this.name = name; + this.pollInterval = pollInterval; + this.msgPackProcessor = msgPackProcessor; + this.consumerExecutor = consumerExecutor; + this.threadPrefix = threadPrefix; + this.consumer = consumerCreator.get(); + } + + public void subscribe() { + consumer.subscribe(); + } + + public void subscribe(Set partitions) { + consumer.subscribe(partitions); + } + + public void launch() { + log.info("[{}] Launching consumer", name); + consumerExecutor.submit(() -> { + if (threadPrefix != null) { + ThingsBoardThreadFactory.addThreadNamePrefix(threadPrefix); + } + try { + consumerLoop(consumer); + } catch (Throwable e) { + log.error("Failure in consumer loop", e); + } + log.info("[{}] Consumer stopped", name); + }); + } + + private void consumerLoop(TbQueueConsumer consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + msgPackProcessor.process(msgs, consumer); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + } + + public void stop() { + log.debug("[{}] Stopping consumer", name); + stopped = true; + consumer.unsubscribe(); + } + + public interface MsgPackProcessor { + void process(List msgs, TbQueueConsumer consumer) throws Exception; + } +} \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 938ef7986e..cbc4e79dd3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1638,11 +1638,13 @@ queue: poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" - # Delay in seconds for starting tasks reprocessing after start-up - reprocessing-start-delay-sec: "${TB_HOUSEKEEPER_REPROCESSING_START_DELAY_SEC:300}" - # Period in seconds of tasks reprocessing - task-reprocessing-delay-sec: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_SEC:3600}" - # Maximum amount of task reprocessing attempts. After exceeding, the task will be ignored until the next service start-up + # Comma-separated list of task types that shouldn't be processed. Available task types: + # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled), + # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS + disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" + # Delay in milliseconds between tasks reprocessing + task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:5000}" + # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" stats: # Enable/disable statistics for Housekeeper diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 44891c2b7f..82c488f0ab 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.housekeeper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; +import net.bytebuddy.implementation.bytecode.Throw; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -104,8 +105,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @DaoSqlTest @TestPropertySource(properties = { "transport.http.enabled=true", - "queue.core.housekeeper.reprocessing-start-delay-sec=1", - "queue.core.housekeeper.task-reprocessing-delay-sec=2", + "queue.core.housekeeper.task-reprocessing-delay-ms=2000", "queue.core.housekeeper.poll-interval-ms=1000", "queue.core.housekeeper.max-reprocessing-attempts=5" }) @@ -258,7 +258,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } @Test - public void whenTaskProcessingFails_thenReprocessUntilSuccessful() throws Exception { + public void whenTaskProcessingFails_thenReprocess() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); @@ -267,8 +267,8 @@ public class HousekeeperServiceTest extends AbstractControllerTest { doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); - int attempts = 3; - await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + int attempts = 2; + await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, 0); for (int i = 1; i <= attempts; i++) { int attempt = i; @@ -279,7 +279,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { }); assertThat(getTimeseriesHistory(device.getId())).isNotEmpty(); - doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); // fixing the code + doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(getTimeseriesHistory(device.getId())).isEmpty(); }); @@ -314,7 +314,12 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } private void verifyTaskProcessing(EntityId entityId, HousekeeperTaskType taskType, int expectedAttempt) throws Exception { - verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); + try { + verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); + } catch (Throwable e) { + e.printStackTrace(); + throw e; + } } private ArgumentMatcher getTaskMatcher(EntityId entityId, HousekeeperTaskType taskType, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index af4e746987..5911418dc3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -19,9 +19,11 @@ import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.ToString; import org.thingsboard.server.common.data.User; @Data +@ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index 515703991c..4156200758 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -20,10 +20,12 @@ import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @Data +@ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java index 59e815bbfa..cef43756e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -19,10 +19,12 @@ import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @Data +@ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java index 1656626e6b..80b8edc345 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -19,10 +19,12 @@ import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @Data +@ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java index e9737bf90c..3b06f0cf3e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -33,11 +34,14 @@ import org.thingsboard.server.queue.provider.TbQueueProducerProvider; @Slf4j public class DefaultHousekeeperClient implements HousekeeperClient { + private final HousekeeperConfig config; private final TbQueueProducer> producer; private final TopicPartitionInfo submitTpi; private final TbQueueCallback submitCallback; - public DefaultHousekeeperClient(TbQueueProducerProvider producerProvider) { + public DefaultHousekeeperClient(HousekeeperConfig config, + TbQueueProducerProvider producerProvider) { + this.config = config; this.producer = producerProvider.getHousekeeperMsgProducer(); this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); this.submitCallback = new TbQueueCallback() { @@ -55,7 +59,13 @@ public class DefaultHousekeeperClient implements HousekeeperClient { @Override public void submitTask(HousekeeperTask task) { - log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getTaskType()); + HousekeeperTaskType taskType = task.getTaskType(); + if (config.getDisabledTaskTypes().contains(taskType)) { + log.trace("Task type {} is disabled, ignoring {}", taskType, task); + return; + } + + log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), taskType); /* * using msg key as entity id so that msgs related to certain entity are pushed to same partition, * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java new file mode 100644 index 0000000000..b03861a7ee --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.housekeeper; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; + +import java.util.Set; + +@Component +@Getter +public class HousekeeperConfig { + + @Value("${queue.core.housekeeper.disabled-task-types:}") + private Set disabledTaskTypes; + @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") + private int taskProcessingTimeout; + @Value("${queue.core.housekeeper.poll-interval-ms:500}") + private int pollInterval; + @Value("${queue.core.housekeeper.task-reprocessing-delay-ms:5000}") + private int taskReprocessingDelay; + @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") + private int maxReprocessingAttempts; + +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java index de097cfd6c..4d0876bf8e 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java @@ -51,6 +51,11 @@ public class ThingsBoardThreadFactory implements ThreadFactory { Thread.currentThread().setName(name); } + public static void addThreadNamePrefix(String prefix) { + String name = Thread.currentThread().getName(); + name = prefix + "-" + name; + Thread.currentThread().setName(name); + } @Override public Thread newThread(Runnable r) { From feb629f2c28f78739b51cf9a76d1217032bebf30 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 11 Apr 2024 14:07:35 +0300 Subject: [PATCH 31/72] Alarms deletion in batches with id offset --- .../housekeeper/HousekeeperService.java | 2 +- .../AlarmsDeletionTaskProcessor.java | 39 +++++++++--- .../housekeeper/HousekeeperServiceTest.java | 60 +++++++++++++------ .../server/dao/alarm/AlarmService.java | 2 - .../AlarmsDeletionHousekeeperTask.java | 46 ++++++++++++++ .../data/housekeeper/HousekeeperTask.java | 3 +- .../server/common/data/page/PageLink.java | 6 ++ .../server/common/data/page/SortOrder.java | 4 ++ .../org/thingsboard/server/dao/DaoUtil.java | 35 ++++++++++- .../server/dao/alarm/BaseAlarmService.java | 16 ----- .../server/dao/sql/alarm/AlarmRepository.java | 11 +++- .../server/dao/sql/alarm/JpaAlarmDao.java | 2 +- 12 files changed, 173 insertions(+), 53 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 89953676c3..b16c1b0de9 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -92,7 +92,7 @@ public class HousekeeperService { private void processMsgs(List> msgs, TbQueueConsumer> consumer) { for (TbProtoQueueMsg msg : msgs) { - log.trace("Processing task: {} attempt={}", msg, msg.getValue().getTask().getAttempt()); + log.trace("Processing task: {}", msg); try { processTask(msg.getValue()); } catch (InterruptedException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index 42e77d9c8e..d775cdc4db 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -19,26 +19,49 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.AlarmsDeletionHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmService; +import java.util.UUID; +import java.util.stream.Collectors; + @Component @RequiredArgsConstructor @Slf4j -public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor { +public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor { private final AlarmService alarmService; @Override - public void process(HousekeeperTask task) throws Exception { - EntityType entityType = task.getEntityId().getEntityType(); + public void process(AlarmsDeletionHousekeeperTask task) throws Exception { + EntityId entityId = task.getEntityId(); + EntityType entityType = entityId.getEntityType(); + TenantId tenantId = task.getTenantId(); + if (entityType == EntityType.DEVICE || entityType == EntityType.ASSET) { - int count = alarmService.deleteAlarmsByOriginatorId(task.getTenantId(), task.getEntityId()); - log.debug("[{}][{}][{}] Deleted {} alarms", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), count); + if (task.getAlarms() == null) { + DaoUtil.iterateWithKeyOffset(pageLink -> { + return alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, pageLink); + }, 128, alarms -> { + housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream() + .map(UUIDBased::getId).collect(Collectors.toList()))); + log.debug("[{}][{}][{}] Submitted task for deleting {} alarms", tenantId, entityType, entityId, alarms.size()); + }); + } else { + for (UUID alarmId : task.getAlarms()) { + alarmService.delAlarm(tenantId, new AlarmId(alarmId)); + } + log.debug("[{}][{}][{}] Deleted {} alarms", tenantId, entityType, entityId, task.getAlarms().size()); + } } else { - int count = alarmService.deleteEntityAlarmRecords(task.getTenantId(), task.getEntityId()); - log.debug("[{}][{}][{}] Deleted {} entity alarms", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), count); + int count = alarmService.deleteEntityAlarmRecords(tenantId, entityId); + log.debug("[{}][{}][{}] Deleted {} entity alarms", tenantId, entityType, entityId, count); } } diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 82c488f0ab..26062810a1 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -17,25 +17,24 @@ package org.thingsboard.server.service.housekeeper; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; -import net.bytebuddy.implementation.bytecode.Throw; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.asset.Asset; @@ -79,7 +78,6 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; -import org.thingsboard.server.service.housekeeper.processor.TelemetryDeletionTaskProcessor; import org.thingsboard.server.service.housekeeper.processor.TsHistoryDeletionTaskProcessor; import java.util.Arrays; @@ -156,7 +154,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenDeviceIsDeleted_thenCleanUpRelatedData() throws Exception { - Device device = createDevice("wekfwepf", "wekfwepf"); + Device device = createDevice("test", "test"); createRelatedData(device.getId()); doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); @@ -187,7 +185,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @Test public void whenUserIsDeleted_thenCleanUpRelatedData() throws Exception { - Device device = createDevice("vneoruvhwe", "vneoruvhwe"); + Device device = createDevice("test", "test"); UserId userId = customerUserId; createRelatedData(userId); Alarm alarm = Alarm.builder() @@ -210,6 +208,20 @@ public class HousekeeperServiceTest extends AbstractControllerTest { }); } + @Test + public void whenDeviceIsDeleted_thenDeleteAllAlarms() throws Exception { + Device device = createDevice("test", "test"); + for (int i = 1; i <= 1000; i++) { + createAlarm(device.getId()); + } + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoAlarms(device.getId()); + }); + } + @Test public void whenTenantIsDeleted_thenDeleteAllEntitiesAndCleanUpRelatedData() throws Exception { loginDifferentTenant(); @@ -220,7 +232,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { createRelatedData(differentTenantCustomerId); loginDifferentTenant(); - Device device = createDevice("oi324rujoi", "oi324rujoi"); + Device device = createDevice("test", "test"); createRelatedData(device.getId()); Asset asset = createAsset(); @@ -258,11 +270,12 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } @Test + @Ignore // FIXME !!! public void whenTaskProcessingFails_thenReprocess() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); - Device device = createDevice("vep9ruv32", "vep9ruv32"); + Device device = createDevice("test", "test"); createRelatedData(device.getId()); doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); @@ -286,11 +299,12 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } @Test + @Ignore // FIXME !!! public void whenReprocessingAttemptsExceeded_thenReprocessOnNextStartUp() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); - Device device = createDevice("woeifjiowejf", "woeifjiowejf"); + Device device = createDevice("test", "test"); createRelatedData(device.getId()); doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); @@ -332,32 +346,31 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private void createRelatedData(EntityId entityId) throws Exception { createTelemetry(entityId); - for (AttributeScope scope : List.of(AttributeScope.SERVER_SCOPE, AttributeScope.SHARED_SCOPE, AttributeScope.CLIENT_SCOPE)) { + for (AttributeScope scope : AttributeScope.values()) { createAttribute(entityId, scope, scope + ATTRIBUTE_KEY); } createEvent(entityId); } private void verifyNoRelatedData(EntityId entityId) throws Exception { - List expectedTaskTypes = List.of(HousekeeperTaskType.DELETE_TELEMETRY, HousekeeperTaskType.DELETE_ATTRIBUTES, HousekeeperTaskType.DELETE_EVENTS, HousekeeperTaskType.DELETE_ALARMS); - for (HousekeeperTaskType taskType : expectedTaskTypes) { - verify(housekeeperClient).submitTask(argThat(task -> task.getTaskType() == taskType && task.getEntityId().equals(entityId))); - } assertThat(entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()).findEntity(tenantId, entityId)).isEmpty(); assertThat(getLatestTelemetry(entityId)).isNull(); assertThat(getTimeseriesHistory(entityId)).isEmpty(); - for (String scope : List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE, DataConstants.CLIENT_SCOPE)) { + for (AttributeScope scope : AttributeScope.values()) { assertThat(attributesService.findAll(tenantId, entityId, scope).get()).isEmpty(); } assertThat(getEvents(entityId)).isEmpty(); assertThat(alarmDao.findEntityAlarmRecordsByEntityId(tenantId, entityId)).isEmpty(); - assertThat(alarmService.findAlarms(tenantId, AlarmQuery.builder().pageLink(new TimePageLink(100)).build()).getData()) - .filteredOn(alarm -> alarm.getOriginator().equals(entityId)).isEmpty(); + verifyNoAlarms(entityId); assertThat(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); assertThat(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); } + private void verifyNoAlarms(EntityId entityId) { + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, new PageLink(1)).getData()).isEmpty(); + } + private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception { attributesService.save(tenantId, entityId, scope, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, KV_VALUE))).get(); } @@ -397,8 +410,17 @@ public class HousekeeperServiceTest extends AbstractControllerTest { List entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId()); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType())); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType())); - assertThat(alarmService.findAlarms(tenantId, AlarmQuery.builder().pageLink(new TimePageLink(100)).build()).getData()) - .filteredOn(a -> a.getOriginator().equals(deviceId)).isNotEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, new PageLink(1)).getData()).isNotEmpty(); + } + + private void createAlarm(DeviceId deviceId) { + Alarm alarm = doPost("/api/alarm", Alarm.builder() + .tenantId(tenantId) + .originator(deviceId) + .severity(AlarmSeverity.CRITICAL) + .type("test alarm for " + deviceId + " " + RandomStringUtils.randomAlphabetic(10)) + .build(), Alarm.class); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, new PageLink(1)).getData()).isNotEmpty(); } private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 57bee7ba1a..32ce3be62c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -81,8 +81,6 @@ public interface AlarmService extends EntityDaoService { void delAlarmTypes(TenantId tenantId, Set types); - int deleteAlarmsByOriginatorId(TenantId tenantId, EntityId entityId); - // Other API Alarm findAlarmById(TenantId tenantId, AlarmId alarmId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java new file mode 100644 index 0000000000..ba9627565d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.UUID; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { + + private List alarms; + + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, List alarms) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); + this.alarms = alarms; + } + + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 08862d26be..344edc3f0d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -37,6 +37,7 @@ import java.io.Serializable; @Type(name = "DELETE_TS_HISTORY", value = TsHistoryDeletionHousekeeperTask.class), @Type(name = "DELETE_LATEST_TS", value = LatestTsDeletionHousekeeperTask.class), @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), + @Type(name = "DELETE_ALARMS", value = AlarmsDeletionHousekeeperTask.class), @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) }) @Data @@ -81,7 +82,7 @@ public class HousekeeperTask implements Serializable { } public static HousekeeperTask deleteAlarms(TenantId tenantId, EntityId entityId) { - return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); + return new AlarmsDeletionHousekeeperTask(tenantId, entityId); } public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java index 042a9b1896..8d806b7006 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java @@ -16,15 +16,20 @@ package org.thingsboard.server.common.data.page; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import org.springframework.data.domain.Sort; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; @Data +@AllArgsConstructor +@Builder public class PageLink { protected static final String DEFAULT_SORT_PROPERTY = "id"; @@ -34,6 +39,7 @@ public class PageLink { private final int pageSize; private final int page; private final SortOrder sortOrder; + private UUID idOffset; public PageLink(PageLink pageLink) { this.pageSize = pageLink.getPageSize(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java index 5cafed6df2..539dd1cb40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java @@ -32,6 +32,10 @@ public class SortOrder { this.direction = direction; } + public static SortOrder of(String property, Direction direction) { + return new SortOrder(property, direction); + } + public static enum Direction { ASC, DESC } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 865306c73b..a0941705eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -18,10 +18,12 @@ package org.thingsboard.server.dao; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.page.PageData; @@ -51,8 +53,17 @@ public abstract class DaoUtil { return new PageData<>(data, page.getTotalPages(), page.getTotalElements(), page.hasNext()); } - public static PageData pageToPageData(Page page) { - return new PageData<>(page.getContent(), page.getTotalPages(), page.getTotalElements(), page.hasNext()); + public static PageData pageToPageData(Slice slice) { + int totalPages; + long totalElements; + if (slice instanceof Page page) { + totalPages = page.getTotalPages(); + totalElements = page.getTotalElements(); + } else { + totalPages = 0; + totalElements = 0; + } + return new PageData<>(slice.getContent(), totalPages, totalElements, slice.hasNext()); } public static Pageable toPageable(PageLink pageLink) { @@ -158,6 +169,26 @@ public abstract class DaoUtil { } while (hasNextBatch); } + public static void iterateWithKeyOffset(Function> findFunction, int pageSize, Consumer> processor) { + List data; + UUID last = null; + while (true) { + PageLink pageLink = PageLink.builder() + .pageSize(pageSize) + .sortOrder(SortOrder.of("id", SortOrder.Direction.ASC)) + .idOffset(last) + .build(); + data = findFunction.apply(pageLink).getData(); + + if (!data.isEmpty()) { + processor.accept(data); + last = data.get(data.size() - 1).getId(); + } else { + break; + } + } + } + public static String getStringId(UUIDBased id) { if (id != null) { return id.toString(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index f9ddc212c8..33b2d5a4c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -51,7 +51,6 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; @@ -215,21 +214,6 @@ public class BaseAlarmService extends AbstractCachedEntityService alarms; - int count = 0; - do { - alarms = alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, pageLink); - for (AlarmId alarmId : alarms.getData()) { - delAlarm(tenantId, alarmId); - count++; - } - } while (alarms.hasNext()); - return count; - } - private List createEntityAlarmRecords(Alarm alarm) throws ExecutionException, InterruptedException { Set propagatedEntitiesSet = new LinkedHashSet<>(); propagatedEntitiesSet.add(alarm.getOriginator()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 83827973ba..ff9a2ed7d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -17,13 +17,13 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.model.sql.AlarmInfoEntity; @@ -333,8 +333,13 @@ public interface AlarmRepository extends JpaRepository { @Query("SELECT a.id FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.assigneeId = :assigneeId") Page findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, @Param("assigneeId") UUID assigneeId, Pageable pageable); - @Query("SELECT a.id FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.originatorId = :originatorId") - Page findAlarmIdsByOriginatorId(@Param("tenantId") UUID tenantId, @Param("originatorId") UUID originatorId, Pageable pageable); + // using Slice so that count query is not executed + @Query(value = "SELECT id FROM alarm WHERE tenant_id = :tenantId AND originator_id = :originatorId " + + "AND (cast(:idOffset as uuid) IS NULL OR id > cast(:idOffset as uuid))", nativeQuery = true) + Slice findAlarmIdsByOriginatorId(@Param("tenantId") UUID tenantId, + @Param("originatorId") UUID originatorId, + @Param("idOffset") UUID idOffset, + Pageable pageable); @Query(value = "SELECT create_or_update_active_alarm(:t_id, :c_id, :a_id, :a_created_ts, :a_o_id, :a_o_type, :a_type, :a_severity, " + ":a_start_ts, :a_end_ts, :a_details, :a_propagate, :a_propagate_to_owner, " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index c326c6452b..6c1118cdba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -298,7 +298,7 @@ public class JpaAlarmDao extends JpaAbstractDao implements A @Override public PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink) { - return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), DaoUtil.toPageable(pageLink))) + return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), pageLink.getIdOffset(), DaoUtil.toPageable(pageLink))) .mapData(AlarmId::new); } From 9064bfaa7663bcde45d4cb121cb2c5bf886f5ea9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 11 Apr 2024 17:18:47 +0300 Subject: [PATCH 32/72] Delete tenant entities in batches with id offset --- .../AlarmsDeletionTaskProcessor.java | 15 +++-- .../EntitiesDeletionTaskProcessor.java | 41 +++++++++++- .../housekeeper/HousekeeperServiceTest.java | 6 +- .../server/dao/alarm/AlarmService.java | 3 +- .../server/common/data/EntityType.java | 67 ++++++++++--------- .../AlarmsDeletionHousekeeperTask.java | 8 +-- .../EntitiesDeletionHousekeeperTask.java | 9 +++ .../server/common/data/page/PageLink.java | 14 ++-- .../java/org/thingsboard/server/dao/Dao.java | 2 + .../org/thingsboard/server/dao/DaoUtil.java | 21 ------ .../server/dao/alarm/AlarmDao.java | 2 +- .../server/dao/alarm/BaseAlarmService.java | 6 +- .../server/dao/entity/EntityDaoRegistry.java | 49 ++++++++++++++ .../server/dao/sql/JpaAbstractDao.java | 19 ++++++ .../server/dao/sql/alarm/JpaAlarmDao.java | 4 +- 15 files changed, 183 insertions(+), 83 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index d775cdc4db..c4834df71d 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -25,9 +25,9 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmService; +import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -46,13 +46,18 @@ public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor { - return alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, pageLink); - }, 128, alarms -> { + AlarmId last = null; + while (true) { + List alarms = alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, entityId, last, 128); + if (alarms.isEmpty()) { + break; + } + housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream() .map(UUIDBased::getId).collect(Collectors.toList()))); + last = alarms.get(alarms.size() - 1); log.debug("[{}][{}][{}] Submitted task for deleting {} alarms", tenantId, entityType, entityId, alarms.size()); - }); + } } else { for (UUID alarmId : task.getAlarms()) { alarmService.delAlarm(tenantId, new AlarmId(alarmId)); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index 95d45a49be..fb01fe4605 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -16,22 +16,59 @@ package org.thingsboard.server.service.housekeeper.processor; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import java.util.List; +import java.util.UUID; + @Component @RequiredArgsConstructor +@Slf4j public class EntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { + private final EntityDaoRegistry entityDaoRegistry; private final EntityServiceRegistry entityServiceRegistry; @Override public void process(EntitiesDeletionHousekeeperTask task) throws Exception { - EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(task.getEntityType()); - entityService.deleteByTenantId(task.getTenantId()); + EntityType entityType = task.getEntityType(); + TenantId tenantId = task.getTenantId(); + Dao entityDao = entityDaoRegistry.getDao(entityType); + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); + + if (task.getEntities() == null) { + UUID last = null; + while (true) { + List entities = entityDao.findIdsByTenantIdAndIdOffset(tenantId, last, 128); + if (entities.isEmpty()) { + break; + } + + housekeeperClient.submitTask(new EntitiesDeletionHousekeeperTask(tenantId, entityType, task.getEntities())); + last = entities.get(entities.size() - 1); + log.debug("[{}] Submitted task for deleting {} {}s", tenantId, entities.size(), entityType.getNormalName().toLowerCase()); + } + } else { + for (UUID entityUuid : task.getEntities()) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// Optional> entity = entityService.findEntity(tenantId, entityId); +// if (entity.isEmpty()) { +// continue; +// } + entityService.deleteEntity(tenantId, entityId); + } + } } @Override diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index 26062810a1..efbfbffbb9 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -368,7 +368,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } private void verifyNoAlarms(EntityId entityId) { - assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, new PageLink(1)).getData()).isEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, entityId, null, 10)).isEmpty(); } private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception { @@ -410,7 +410,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { List entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId()); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType())); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType())); - assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, new PageLink(1)).getData()).isNotEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, deviceId, null, 10)).isNotEmpty(); } private void createAlarm(DeviceId deviceId) { @@ -420,7 +420,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { .severity(AlarmSeverity.CRITICAL) .type("test alarm for " + deviceId + " " + RandomStringUtils.randomAlphabetic(10)) .build(), Alarm.class); - assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, new PageLink(1)).getData()).isNotEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, deviceId, null, 10)).isNotEmpty(); } private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index 32ce3be62c..ee584ff856 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.Collection; +import java.util.List; import java.util.Set; @@ -106,7 +107,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink); + List findAlarmIdsByOriginatorIdAndIdOffset(TenantId tenantId, EntityId originatorId, AlarmId idOffset, int limit); int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index bb43bf4d11..6748aebb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.EnumSet; import java.util.List; -import java.util.stream.Collectors; /** * @author Andrew Shvayka @@ -28,51 +27,57 @@ import java.util.stream.Collectors; public enum EntityType { TENANT(1), CUSTOMER(2), - USER(3), + USER(3, "tb_user"), DASHBOARD(4), ASSET(5), DEVICE(6), - ALARM (7), - RULE_CHAIN (11), - RULE_NODE (12), - - ENTITY_VIEW (15) { + ALARM(7), + RULE_CHAIN(11), + RULE_NODE(12), + ENTITY_VIEW(15) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override - public String getNormalName () { + public String getNormalName() { return "Entity View"; } }, - WIDGETS_BUNDLE (16), - WIDGET_TYPE (17), - TENANT_PROFILE (20), - DEVICE_PROFILE (21), - ASSET_PROFILE (22), - API_USAGE_STATE (23), - TB_RESOURCE (24), - OTA_PACKAGE (25), - EDGE (26), - RPC (27), - QUEUE (28), - NOTIFICATION_TARGET (29), - NOTIFICATION_TEMPLATE (30), - NOTIFICATION_REQUEST (31), - NOTIFICATION (32), - NOTIFICATION_RULE (33), + WIDGETS_BUNDLE(16), + WIDGET_TYPE(17), + TENANT_PROFILE(20), + DEVICE_PROFILE(21), + ASSET_PROFILE(22), + API_USAGE_STATE(23), + TB_RESOURCE(24, "resource"), + OTA_PACKAGE(25), + EDGE(26), + RPC(27), + QUEUE(28), + NOTIFICATION_TARGET(29), + NOTIFICATION_TEMPLATE(30), + NOTIFICATION_REQUEST(31), + NOTIFICATION(32), + NOTIFICATION_RULE(33), QUEUE_STATS(34); @Getter private final int protoNumber; // Corresponds to EntityTypeProto + @Getter + private final String tableName; + @Getter + private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_") + .toLowerCase().replaceAll("_", " ")); + + public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() + .map(EntityType::getNormalName).toList(); - private EntityType(int protoNumber) { + EntityType(int protoNumber) { this.protoNumber = protoNumber; + this.tableName = name().toLowerCase(); } - public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() - .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList()); - - @Getter - private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_") - .toLowerCase().replaceAll("_", " ")); + EntityType(int protoNumber, String tableName) { + this.protoNumber = protoNumber; + this.tableName = tableName; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java index ba9627565d..9bd8f29bb4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -34,13 +34,13 @@ public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { private List alarms; - public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, List alarms) { - super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); - this.alarms = alarms; + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { + this(tenantId, entityId, null); } - public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, List alarms) { super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); + this.alarms = alarms; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index 4156200758..e29c3d3c51 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -24,6 +24,9 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.util.List; +import java.util.UUID; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @@ -31,10 +34,16 @@ import org.thingsboard.server.common.data.id.TenantId; public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private EntityType entityType; + private List entities; protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { + this(tenantId, entityType, null); + } + + public EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType, List entities) { super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); this.entityType = entityType; + this.entities = entities; } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java index 8d806b7006..d6a3b5ad7d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java @@ -16,20 +16,15 @@ package org.thingsboard.server.common.data.page; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; import org.springframework.data.domain.Sort; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.stream.Collectors; @Data -@AllArgsConstructor -@Builder public class PageLink { protected static final String DEFAULT_SORT_PROPERTY = "id"; @@ -39,7 +34,6 @@ public class PageLink { private final int pageSize; private final int page; private final SortOrder sortOrder; - private UUID idOffset; public PageLink(PageLink pageLink) { this.pageSize = pageLink.getPageSize(); @@ -69,10 +63,10 @@ public class PageLink { @JsonIgnore public PageLink nextPageLink() { - return new PageLink(this.pageSize, this.page+1, this.textSearch, this.sortOrder); + return new PageLink(this.pageSize, this.page + 1, this.textSearch, this.sortOrder); } - public Sort toSort(SortOrder sortOrder, Map columnMap, boolean addDefaultSorting) { + public Sort toSort(SortOrder sortOrder, Map columnMap, boolean addDefaultSorting) { if (sortOrder == null) { return DEFAULT_SORT; } else { @@ -80,7 +74,7 @@ public class PageLink { } } - public Sort toSort(List sortOrders, Map columnMap, boolean addDefaultSorting) { + public Sort toSort(List sortOrders, Map columnMap, boolean addDefaultSorting) { if (addDefaultSorting && !isDefaultSortOrderAvailable(sortOrders)) { sortOrders = new ArrayList<>(sortOrders); sortOrders.add(new SortOrder(DEFAULT_SORT_PROPERTY, SortOrder.Direction.ASC)); @@ -88,7 +82,7 @@ public class PageLink { return Sort.by(sortOrders.stream().map(s -> toSortOrder(s, columnMap)).collect(Collectors.toList())); } - private Sort.Order toSortOrder(SortOrder sortOrder, Map columnMap) { + private Sort.Order toSortOrder(SortOrder sortOrder, Map columnMap) { String property = sortOrder.getProperty(); if (columnMap.containsKey(property)) { property = columnMap.get(property); diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 8d408945fa..e7290cdced 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -43,6 +43,8 @@ public interface Dao { void removeAllByIds(Collection ids); + List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); + default EntityType getEntityType() { return null; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index a0941705eb..01a72ea58c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -23,7 +23,6 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.page.PageData; @@ -169,26 +168,6 @@ public abstract class DaoUtil { } while (hasNextBatch); } - public static void iterateWithKeyOffset(Function> findFunction, int pageSize, Consumer> processor) { - List data; - UUID last = null; - while (true) { - PageLink pageLink = PageLink.builder() - .pageSize(pageSize) - .sortOrder(SortOrder.of("id", SortOrder.Direction.ASC)) - .idOffset(last) - .build(); - data = findFunction.apply(pageLink).getData(); - - if (!data.isEmpty()) { - processor.accept(data); - last = data.get(data.size() - 1).getId(); - } else { - break; - } - } - } - public static String getStringId(UUIDBased id) { if (id != null) { return id.toString(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 365a5ac2e0..03038b3bfe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -80,7 +80,7 @@ public interface AlarmDao extends Dao { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UUID userId, PageLink pageLink); - PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink); + PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, AlarmId idOffset, PageLink pageLink); void createEntityAlarmRecord(EntityAlarm entityAlarm); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 33b2d5a4c3..9d0940a605 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -313,9 +313,9 @@ public class BaseAlarmService extends AbstractCachedEntityService findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink) { - log.trace("[{}] Executing findAlarmsByOriginatorId [{}]", tenantId, originatorId); - return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, pageLink); + public List findAlarmIdsByOriginatorIdAndIdOffset(TenantId tenantId, EntityId originatorId, AlarmId idOffset, int limit) { + log.trace("[{}] Executing findAlarmIdsByOriginatorIdAndIdOffset [{}][{}]", tenantId, originatorId, idOffset); + return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, idOffset, new PageLink(limit)).getData(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java new file mode 100644 index 0000000000..261454966c --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entity; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.Dao; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@Slf4j +public class EntityDaoRegistry { + + private final Map> daos; + + private EntityDaoRegistry(List> daos) { + this.daos = daos.stream().filter(dao -> dao.getEntityType() != null) + .collect(Collectors.toMap(Dao::getEntityType, dao -> dao)); + } + + @SuppressWarnings("unchecked") + public Dao getDao(EntityType entityType) { + Dao dao = (Dao) daos.get(entityType); + if (dao == null) { + throw new IllegalArgumentException("Missing dao for entity type " + entityType); + } + return dao; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index fdb96195fb..6bfd74ac87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -19,7 +19,9 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; @@ -41,6 +43,9 @@ public abstract class JpaAbstractDao, D> extends JpaAbstractDaoListeningExecutorService implements Dao { + @Autowired + protected JdbcTemplate jdbcTemplate; + protected abstract Class getEntityClass(); protected abstract JpaRepository getRepository(); @@ -117,6 +122,20 @@ public abstract class JpaAbstractDao, D> ids.forEach(repository::deleteById); } + @Override + public List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit) { + String tableName = Optional.ofNullable(getEntityType()) + .orElseThrow(() -> new IllegalArgumentException("Entity type not specified")) + .getTableName(); + if (idOffset == null) { + return jdbcTemplate.queryForList("SELECT id FROM " + tableName + " WHERE tenant_id = ? ORDER BY id LIMIT ?", + UUID.class, tenantId.getId(), limit); + } else { + return jdbcTemplate.queryForList("SELECT id FROM " + tableName + " WHERE tenant_id = ? AND id > ? ORDER BY id LIMIT ?", + UUID.class, tenantId.getId(), idOffset, limit); + } + } + @Override public List find(TenantId tenantId) { List entities = Lists.newArrayList(getRepository().findAll()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 6c1118cdba..da603a6c6f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -297,8 +297,8 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink) { - return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), pageLink.getIdOffset(), DaoUtil.toPageable(pageLink))) + public PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, AlarmId idOffset, PageLink pageLink) { + return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), idOffset != null ? idOffset.getId() : null, DaoUtil.toPageable(pageLink))) .mapData(AlarmId::new); } From 0b18de1482ba6769dfca8bd5116f0a94f2acc0c8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 12 Apr 2024 15:32:07 +0300 Subject: [PATCH 33/72] Refactor entities deletion --- .../AlarmsDeletionTaskProcessor.java | 6 +- .../EntitiesDeletionTaskProcessor.java | 9 +-- .../DefaultExportableEntitiesService.java | 2 +- .../server/dao/entity/EntityDaoService.java | 2 +- .../server/dao/user/UserService.java | 2 + .../housekeeper/DefaultHousekeeperClient.java | 2 +- .../dao/asset/AssetProfileServiceImpl.java | 21 ++++--- .../server/dao/asset/BaseAssetService.java | 19 ++++--- .../dao/customer/CustomerServiceImpl.java | 19 ++++--- .../dao/dashboard/DashboardServiceImpl.java | 12 ++-- .../dao/device/DeviceProfileServiceImpl.java | 21 ++++--- .../server/dao/device/DeviceServiceImpl.java | 26 +++++---- .../server/dao/edge/EdgeServiceImpl.java | 12 +++- .../server/dao/entity/EntityDaoRegistry.java | 2 - .../dao/entityview/EntityViewServiceImpl.java | 9 +++ .../DefaultNotificationRequestService.java | 10 ++++ .../DefaultNotificationRuleService.java | 10 ++-- .../DefaultNotificationTargetService.java | 15 ++--- .../DefaultNotificationTemplateService.java | 12 ++-- .../server/dao/ota/BaseOtaPackageService.java | 5 ++ .../server/dao/queue/BaseQueueService.java | 5 ++ .../dao/queue/BaseQueueStatsService.java | 6 ++ .../dao/resource/BaseResourceService.java | 10 ++-- .../server/dao/rpc/BaseRpcService.java | 5 ++ .../server/dao/rule/BaseRuleChainService.java | 17 ++++-- .../server/dao/sql/queue/JpaQueueDao.java | 9 ++- .../dao/sql/queue/JpaQueueStatsDao.java | 8 ++- .../dao/sql/tenant/JpaTenantProfileDao.java | 7 ++- .../server/dao/tenant/TenantServiceImpl.java | 2 +- .../usagerecord/ApiUsageStateServiceImpl.java | 10 ++++ .../server/dao/user/UserServiceImpl.java | 7 ++- .../dao/widget/WidgetTypeServiceImpl.java | 5 ++ .../dao/widget/WidgetsBundleServiceImpl.java | 22 +++++--- .../dao/service/EntityDaoRegistryTest.java | 55 +++++++++++++++++++ 34 files changed, 274 insertions(+), 110 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index c4834df71d..09fa0977d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -64,10 +64,10 @@ public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor> entity = entityService.findEntity(tenantId, entityId); -// if (entity.isEmpty()) { -// continue; -// } - entityService.deleteEntity(tenantId, entityId); + entityService.deleteEntity(tenantId, entityId, true); } + log.debug("[{}] Deleted {} {}s", tenantId, task.getEntities().size(), entityType.getNormalName().toLowerCase()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 6a2a5b4889..49314e62fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -147,7 +147,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi public void removeById(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); - entityService.deleteEntity(tenantId, id); + entityService.deleteEntity(tenantId, id, false); } private > ExportableEntityDao getExportableEntityDao(EntityType entityType) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java index 992c4fad94..cf55cf1054 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java @@ -30,7 +30,7 @@ public interface EntityDaoService { throw new IllegalArgumentException("Not implemented for " + getEntityType()); } - default void deleteEntity(TenantId tenantId, EntityId id) { + default void deleteEntity(TenantId tenantId, EntityId id, boolean force) { throw new IllegalArgumentException(getEntityType().getNormalName() + " deletion not supported"); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index f137bcc334..0cad776e78 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -77,6 +77,8 @@ public interface UserService extends EntityDaoService { void deleteTenantAdmins(TenantId tenantId); + void deleteAllByTenantId(TenantId tenantId); + PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink); PageData findUsersByCustomerIds(TenantId tenantId, List customerIds, PageLink pageLink); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java index 3b06f0cf3e..ce5311b0e7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java @@ -65,7 +65,7 @@ public class DefaultHousekeeperClient implements HousekeeperClient { return; } - log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), taskType); + log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); /* * using msg key as entity id so that msgs related to certain entity are pushed to same partition, * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index 7946448666..6848c58d5c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -180,11 +180,20 @@ public class AssetProfileServiceImpl extends AbstractCachedEntityService INCORRECT_ASSET_PROFILE_ID + id); - AssetProfile assetProfile = assetProfileDao.findById(tenantId, assetProfileId.getId()); - if (assetProfile != null && assetProfile.isDefault()) { + deleteEntity(tenantId, assetProfileId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + AssetProfile assetProfile = assetProfileDao.findById(tenantId, id.getId()); + if (assetProfile == null) { + return; + } + if (!force && assetProfile.isDefault()) { throw new DataValidationException("Deletion of Default Asset Profile is prohibited!"); } - this.removeAssetProfile(tenantId, assetProfile); + removeAssetProfile(tenantId, assetProfile); } private void removeAssetProfile(TenantId tenantId, AssetProfile assetProfile) { @@ -312,12 +321,6 @@ public class AssetProfileServiceImpl extends AbstractCachedEntityService INCORRECT_ASSET_ID + id); - if (entityViewService.existsByTenantIdAndEntityId(tenantId, assetId)) { + deleteEntity(tenantId, assetId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && entityViewService.existsByTenantIdAndEntityId(tenantId, id)) { throw new DataValidationException("Can't delete asset that has entity views!"); } - Asset asset = assetDao.findById(tenantId, assetId.getId()); + Asset asset = assetDao.findById(tenantId, id.getId()); + if (asset == null) { + return; + } deleteAsset(tenantId, asset); } @@ -473,12 +482,6 @@ public class BaseAssetService extends AbstractCachedEntityService INCORRECT_CUSTOMER_ID + id); + deleteEntity(tenantId, customerId, false); + } + + @Transactional + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CustomerId customerId = (CustomerId) id; Customer customer = findCustomerById(tenantId, customerId); if (customer == null) { - throw new IncorrectParameterException("Unable to delete non-existent customer."); + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent customer."); + } } dashboardService.unassignCustomerDashboards(tenantId, customerId); entityViewService.unassignCustomerEntityViews(customer.getTenantId(), customerId); @@ -204,12 +215,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom return Optional.ofNullable(findCustomerById(tenantId, new CustomerId(entityId.getId()))); } - @Transactional - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteCustomer(tenantId, (CustomerId) id); - } - @Override public long countByTenantId(TenantId tenantId) { return customerDao.countByTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index 8eef618382..167ae51fc3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -245,6 +245,12 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb } } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteDashboard(tenantId, (DashboardId) id); + } + @Override public PageData findDashboardsByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findDashboardsByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); @@ -394,12 +400,6 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardDao.countByTenantId(tenantId); } - @Override - @Transactional - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteDashboard(tenantId, (DashboardId) id); - } - @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 700d8b52f0..7a0f7a56c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -217,11 +217,20 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService INCORRECT_DEVICE_PROFILE_ID + id); - DeviceProfile deviceProfile = deviceProfileDao.findById(tenantId, deviceProfileId.getId()); - if (deviceProfile != null && deviceProfile.isDefault()) { + deleteEntity(tenantId, deviceProfileId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + DeviceProfile deviceProfile = deviceProfileDao.findById(tenantId, id.getId()); + if (deviceProfile == null) { + return; + } + if (!force && deviceProfile.isDefault()) { throw new DataValidationException("Deletion of Default Device Profile is prohibited!"); } - this.removeDeviceProfile(tenantId, deviceProfile); + removeDeviceProfile(tenantId, deviceProfile); } private void removeDeviceProfile(TenantId tenantId, DeviceProfile deviceProfile) { @@ -361,12 +370,6 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService INCORRECT_DEVICE_ID + id); - if (entityViewService.existsByTenantIdAndEntityId(tenantId, deviceId)) { + deleteEntity(tenantId, deviceId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && entityViewService.existsByTenantIdAndEntityId(tenantId, id)) { throw new DataValidationException("Can't delete device that has entity views!"); } - Device device = deviceDao.findById(tenantId, deviceId.getId()); + Device device = deviceDao.findById(tenantId, id.getId()); + if (device == null) { + return; + } deleteDevice(tenantId, device); } @@ -344,12 +353,11 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findDevicesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); - validateId(tenantId, id ->INCORRECT_TENANT_ID + id); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validatePageLink(pageLink); return deviceDao.findDevicesByTenantId(tenantId.getId(), pageLink); } - @Override public PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink) { log.trace("Executing findDeviceInfosByFilter, filter [{}], pageLink [{}]", filter, pageLink); @@ -410,14 +418,14 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByIds(List deviceIds) { log.trace("Executing findDevicesByIdsAsync, deviceIds [{}]", deviceIds); - validateIds(deviceIds, ids-> "Incorrect deviceIds " + ids); + validateIds(deviceIds, ids -> "Incorrect deviceIds " + ids); return deviceDao.findDevicesByIds(toUUIDs(deviceIds)); } @Override public ListenableFuture> findDevicesByIdsAsync(List deviceIds) { log.trace("Executing findDevicesByIdsAsync, deviceIds [{}]", deviceIds); - validateIds(deviceIds, ids-> "Incorrect deviceIds " + ids); + validateIds(deviceIds, ids -> "Incorrect deviceIds " + ids); return deviceDao.findDevicesByIdsAsync(toUUIDs(deviceIds)); } @@ -651,12 +659,6 @@ public class DeviceServiceImpl extends AbstractCachedEntityService tenantDevicesRemover = new PaginatedRemover<>() { @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 1fd3cacb6d..d5b7955d85 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.Nullable; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; @@ -61,7 +62,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -72,7 +72,6 @@ import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.user.UserService; -import jakarta.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -225,6 +224,9 @@ public class EdgeServiceImpl extends AbstractCachedEntityService INCORRECT_EDGE_ID + id); Edge edge = edgeDao.findById(tenantId, edgeId.getId()); + if (edge == null) { + return; + } edgeDao.removeById(tenantId, edgeId.getId()); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(edgeId).build()); @@ -232,6 +234,12 @@ public class EdgeServiceImpl extends AbstractCachedEntityService findEdgesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findEdgesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 261454966c..10e301cce6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.entity; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; @@ -26,7 +25,6 @@ import java.util.Map; import java.util.stream.Collectors; @Service -@RequiredArgsConstructor @Slf4j public class EntityDaoRegistry { diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index faf70654cb..e56055b614 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -328,11 +328,20 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService INCORRECT_ENTITY_VIEW_ID + id); EntityView entityView = entityViewDao.findById(tenantId, entityViewId.getId()); + if (entityView == null) { + return; + } entityViewDao.removeById(tenantId, entityViewId.getId()); publishEvictEvent(new EntityViewEvictEvent(entityView.getTenantId(), entityView.getId(), entityView.getEntityId(), null, entityView.getName(), null)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(entityViewId).build()); } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteEntityView(tenantId, (EntityViewId) id); + } + @Override public void deleteEntityViewsByTenantId(TenantId tenantId) { log.trace("Executing deleteEntityViewsByTenantId, tenantId [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index a74e9484c0..ebeb39ed91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -93,6 +93,16 @@ public class DefaultNotificationRequestService implements NotificationRequestSer eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(request).entityId(request.getId()).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (force) { + notificationRequestDao.removeById(tenantId, id.getId()); + } else { + NotificationRequest notificationRequest = findNotificationRequestById(tenantId, (NotificationRequestId) id); + deleteNotificationRequest(tenantId, notificationRequest); + } + } + @Override public PageData findScheduledNotificationRequests(PageLink pageLink) { return notificationRequestDao.findAllByStatus(NotificationRequestStatus.SCHEDULED, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java index f306818927..ccf8d83887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java @@ -94,6 +94,11 @@ public class DefaultNotificationRuleService extends AbstractEntityService implem eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteNotificationRuleById(tenantId, (NotificationRuleId) id); + } + @Override public void deleteNotificationRulesByTenantId(TenantId tenantId) { notificationRuleDao.removeByTenantId(tenantId); @@ -109,11 +114,6 @@ public class DefaultNotificationRuleService extends AbstractEntityService implem return Optional.ofNullable(findNotificationRuleById(tenantId, new NotificationRuleId(entityId.getId()))); } - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationRuleById(tenantId, (NotificationRuleId) id); - } - @Override public EntityType getEntityType() { return EntityType.NOTIFICATION_RULE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java index 5996ec6c90..3038578e99 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java @@ -187,10 +187,16 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl @Override public void deleteNotificationTargetById(TenantId tenantId, NotificationTargetId id) { - if (notificationRequestDao.existsByTenantIdAndStatusAndTargetId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { + deleteEntity(tenantId, id, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + NotificationTargetId targetId = (NotificationTargetId) id; + if (!force && notificationRequestDao.existsByTenantIdAndStatusAndTargetId(tenantId, NotificationRequestStatus.SCHEDULED, targetId)) { throw new IllegalArgumentException("Recipients group is referenced by scheduled notification request"); } - if (notificationRuleDao.existsByTenantIdAndTargetId(tenantId, id)) { + if (!force && notificationRuleDao.existsByTenantIdAndTargetId(tenantId, targetId)) { throw new IllegalArgumentException("Recipients group is being used in notification rule"); } notificationTargetDao.removeById(tenantId, id.getId()); @@ -216,11 +222,6 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl return Optional.ofNullable(findNotificationTargetById(tenantId, new NotificationTargetId(entityId.getId()))); } - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationTargetById(tenantId, (NotificationTargetId) id); - } - @Override public EntityType getEntityType() { return EntityType.NOTIFICATION_TARGET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index bec6c3d61c..f29894ee9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -76,7 +76,12 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im @Override public void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id) { - if (notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { + deleteEntity(tenantId, id, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, (NotificationTemplateId) id)) { throw new IllegalArgumentException("Notification template is referenced by scheduled notification request"); } try { @@ -104,11 +109,6 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im return Optional.ofNullable(findNotificationTemplateById(tenantId, new NotificationTemplateId(entityId.getId()))); } - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationTemplateById(tenantId, (NotificationTemplateId) id); - } - @Override public EntityType getEntityType() { return EntityType.NOTIFICATION_TEMPLATE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 9a29845c66..06c449880d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -218,6 +218,11 @@ public class BaseOtaPackageService extends AbstractCachedEntityService findQueuesByTenantId(TenantId tenantId) { log.trace("Executing findQueues, tenantId: [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 2274a782bb..6352ac1a4c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -78,6 +78,11 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu queueStatsDao.deleteByTenantId(tenantId); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + queueStatsDao.removeById(tenantId, id.getId()); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findQueueStatsById(tenantId, new QueueStatsId(entityId.getId()))); @@ -87,4 +92,5 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public EntityType getEntityType() { return EntityType.QUEUE_STATS; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index ad35ccd9b1..723d0e8861 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -159,6 +159,11 @@ public class BaseResourceService extends AbstractCachedEntityService findAllTenantResourcesByTenantId(TbResourceInfoFilter filter, PageLink pageLink) { TenantId tenantId = filter.getTenantId(); @@ -213,11 +218,6 @@ public class BaseResourceService extends AbstractCachedEntityService implements Q return DaoUtil.toPageData(queueRepository .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } -} \ No newline at end of file + + @Override + public EntityType getEntityType() { + return EntityType.QUEUE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index ac57d54e90..1cfdbf0b38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.DaoUtil; @@ -63,4 +64,9 @@ public class JpaQueueStatsDao extends JpaAbstractDao INCORRECT_TENANT_ID + id); - userService.deleteByTenantId(tenantId); + userService.deleteAllByTenantId(tenantId); adminSettingsService.deleteAdminSettingsByTenantId(tenantId); notificationSettingsService.deleteNotificationSettings(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 6fe94e6341..d901ae950e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -77,12 +77,22 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A log.trace("Executing deleteApiUsageStateByEntityId [{}]", entityId); validateId(entityId.getId(), id -> "Invalid entity id " + id); ApiUsageState apiUsageState = findApiUsageStateByEntityId(entityId); + deleteApiUsageState(apiUsageState); + } + + private void deleteApiUsageState(ApiUsageState apiUsageState) { if (apiUsageState != null) { apiUsageStateDao.removeById(apiUsageState.getTenantId(), apiUsageState.getUuidId()); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(apiUsageState.getTenantId()).entityId(apiUsageState.getId()).build()); } } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + ApiUsageState apiUsageState = findApiUsageStateById(tenantId, (ApiUsageStateId) id); + deleteApiUsageState(apiUsageState); + } + @Override public ApiUsageState createDefaultApiUsageState(TenantId tenantId, EntityId entityId) { entityId = Objects.requireNonNullElse(entityId, tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 8403520e04..90084e1a3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -322,11 +322,16 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic } @Override - public void deleteByTenantId(TenantId tenantId) { + public void deleteAllByTenantId(TenantId tenantId) { log.trace("Executing deleteByTenantId, tenantId [{}]", tenantId); usersRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAllByTenantId(tenantId); + } + @Override public PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findCustomerUsers, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 8d24746929..f2506a2d9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -115,6 +115,11 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(widgetTypeId).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteWidgetType(tenantId, (WidgetTypeId) id); + } + @Override public PageData findSystemWidgetTypesByPageLink(TenantId tenantId, boolean fullSearch, DeprecatedFilter deprecatedFilter, List widgetTypes, PageLink pageLink) { log.trace("Executing findSystemWidgetTypesByPageLink, fullSearch [{}], deprecatedFilter [{}], widgetTypes [{}], pageLink [{}]", fullSearch, deprecatedFilter, widgetTypes, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 3d16bb519b..ae53c70ec8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -92,12 +92,21 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { public void deleteWidgetsBundle(TenantId tenantId, WidgetsBundleId widgetsBundleId) { log.trace("Executing deleteWidgetsBundle [{}]", widgetsBundleId); Validator.validateId(widgetsBundleId, id -> "Incorrect widgetsBundleId " + id); - WidgetsBundle widgetsBundle = findWidgetsBundleById(tenantId, widgetsBundleId); + deleteEntity(tenantId, widgetsBundleId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + WidgetsBundle widgetsBundle = findWidgetsBundleById(tenantId, (WidgetsBundleId) id); if (widgetsBundle == null) { - throw new IncorrectParameterException("Unable to delete non-existent widgets bundle."); + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent widgets bundle."); + } } - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(widgetsBundleId).build()); - widgetsBundleDao.removeById(tenantId, widgetsBundleId.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); + widgetsBundleDao.removeById(tenantId, id.getId()); } @Override @@ -189,11 +198,6 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId()))); } - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteWidgetsBundle(tenantId, (WidgetsBundleId) id); - } - @Override public EntityType getEntityType() { return EntityType.WIDGETS_BUNDLE; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java new file mode 100644 index 0000000000..08d23db022 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +@Slf4j +@DaoSqlTest +public class EntityDaoRegistryTest extends AbstractServiceTest { + + @Autowired + EntityDaoRegistry entityDaoRegistry; + + @Test + public void givenAllEntityTypes_whenGetDao_thenAllPresent() { + for (EntityType entityType : EntityType.values()) { + Dao dao = assertDoesNotThrow(() -> entityDaoRegistry.getDao(entityType)); + assertThat(dao).isNotNull(); + } + } + + @Test + public void givenAllDaos_whenFindById_thenOk() { + for (EntityType entityType : EntityType.values()) { + assertDoesNotThrow(() -> { + entityDaoRegistry.getDao(entityType).findById(TenantId.SYS_TENANT_ID, UUID.randomUUID()); + }); + } + } + +} From 40d9789035007eaee639b438fc1d8a6176d7250b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 12 Apr 2024 17:24:06 +0300 Subject: [PATCH 34/72] More tests --- .../housekeeper/HousekeeperServiceTest.java | 19 +++++++++---- .../server/dao/sql/JpaAbstractDao.java | 28 +++++++++++-------- .../dao/service/EntityDaoRegistryTest.java | 23 ++++++++++++++- 3 files changed, 52 insertions(+), 18 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index efbfbffbb9..a1015c4a4c 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -80,6 +80,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.service.housekeeper.processor.TsHistoryDeletionTaskProcessor; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -232,13 +233,17 @@ public class HousekeeperServiceTest extends AbstractControllerTest { createRelatedData(differentTenantCustomerId); loginDifferentTenant(); - Device device = createDevice("test", "test"); - createRelatedData(device.getId()); + List devices = new ArrayList<>(); + for (int i = 1; i <= 300; i++) { + Device device = createDevice("test" + i, "test" + i); + createRelatedData(device.getId()); + devices.add(device.getId()); + } Asset asset = createAsset(); createRelatedData(asset.getId()); - createRelation(device.getId(), asset.getId()); - createAlarm(device.getId(), asset.getId()); + createRelation(devices.get(0), asset.getId()); + createAlarm(devices.get(0), asset.getId()); RuleChainMetaData ruleChainMetaData = createRuleChain(); RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); @@ -256,8 +261,10 @@ public class HousekeeperServiceTest extends AbstractControllerTest { loginSysAdmin(); deleteDifferentTenant(); - await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - verifyNoRelatedData(device.getId()); + await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + for (DeviceId deviceId : devices) { + verifyNoRelatedData(deviceId); + } verifyNoRelatedData(asset.getId()); verifyNoRelatedData(ruleNode1Id); verifyNoRelatedData(ruleNode2Id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 6bfd74ac87..973cb4b5b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.SqlDao; import java.util.Collection; @@ -122,24 +123,29 @@ public abstract class JpaAbstractDao, D> ids.forEach(repository::deleteById); } + @Override + public List find(TenantId tenantId) { + List entities = Lists.newArrayList(getRepository().findAll()); + return DaoUtil.convertDataList(entities); + } + @Override public List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit) { - String tableName = Optional.ofNullable(getEntityType()) - .orElseThrow(() -> new IllegalArgumentException("Entity type not specified")) - .getTableName(); + String query = "SELECT id FROM " + getEntityType().getTableName() + " WHERE " + getTenantIdColumn() + " = ? "; + Object[] params; if (idOffset == null) { - return jdbcTemplate.queryForList("SELECT id FROM " + tableName + " WHERE tenant_id = ? ORDER BY id LIMIT ?", - UUID.class, tenantId.getId(), limit); + params = new Object[]{tenantId.getId(), limit}; } else { - return jdbcTemplate.queryForList("SELECT id FROM " + tableName + " WHERE tenant_id = ? AND id > ? ORDER BY id LIMIT ?", - UUID.class, tenantId.getId(), idOffset, limit); + query += " AND id > ? "; + params = new Object[]{tenantId.getId(), idOffset, limit}; } + query += " ORDER BY id LIMIT ?"; + + return jdbcTemplate.queryForList(query, UUID.class, params); } - @Override - public List find(TenantId tenantId) { - List entities = Lists.newArrayList(getRepository().findAll()); - return DaoUtil.convertDataList(entities); + protected String getTenantIdColumn() { + return ModelConstants.TENANT_ID_COLUMN; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java index 08d23db022..8524efc4a0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.service; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.EntityType; @@ -26,6 +27,7 @@ import org.thingsboard.server.dao.entity.EntityDaoRegistry; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @Slf4j @@ -46,10 +48,29 @@ public class EntityDaoRegistryTest extends AbstractServiceTest { @Test public void givenAllDaos_whenFindById_thenOk() { for (EntityType entityType : EntityType.values()) { + Dao dao = entityDaoRegistry.getDao(entityType); assertDoesNotThrow(() -> { - entityDaoRegistry.getDao(entityType).findById(TenantId.SYS_TENANT_ID, UUID.randomUUID()); + dao.findById(TenantId.SYS_TENANT_ID, UUID.randomUUID()); }); } } + @Test + public void givenAllDaos_whenFindIdsByTenantIdAndIdOffset_thenOk() { + for (EntityType entityType : EntityType.values()) { + Dao dao = entityDaoRegistry.getDao(entityType); + try { + dao.findIdsByTenantIdAndIdOffset(TenantId.SYS_TENANT_ID, null, 10); + dao.findIdsByTenantIdAndIdOffset(TenantId.SYS_TENANT_ID, UUID.randomUUID(), 10); + } catch (Exception e) { + String error = ExceptionUtils.getRootCauseMessage(e); + if (error.contains("tenant_id")) { + log.debug("[{}] Ignoring not found tenant_id column", entityType); + } else { + fail("findIdsByTenantIdAndIdOffset for " + entityType + " dao threw error: " + error); + } + } + } + } + } From 881a8048ce7cbb598a410ad8cc5340fe22bbe9b9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 15 Apr 2024 12:34:16 +0300 Subject: [PATCH 35/72] Fix EntityDaoRegistry init --- application/src/main/resources/thingsboard.yml | 2 +- .../server/queue/housekeeper/HousekeeperConfig.java | 2 +- .../server/dao/entity/EntityDaoRegistry.java | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cbc4e79dd3..024eb54974 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1643,7 +1643,7 @@ queue: # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" # Delay in milliseconds between tasks reprocessing - task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:5000}" + task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}" # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" stats: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java index b03861a7ee..4ca3776d4d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java @@ -32,7 +32,7 @@ public class HousekeeperConfig { private int taskProcessingTimeout; @Value("${queue.core.housekeeper.poll-interval-ms:500}") private int pollInterval; - @Value("${queue.core.housekeeper.task-reprocessing-delay-ms:5000}") + @Value("${queue.core.housekeeper.task-reprocessing-delay-ms:3000}") private int taskReprocessingDelay; @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") private int maxReprocessingAttempts; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java index 10e301cce6..7b2f2d42c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -20,19 +20,23 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.Dao; +import java.util.EnumMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Service @Slf4j public class EntityDaoRegistry { - private final Map> daos; + private final Map> daos = new EnumMap<>(EntityType.class); private EntityDaoRegistry(List> daos) { - this.daos = daos.stream().filter(dao -> dao.getEntityType() != null) - .collect(Collectors.toMap(Dao::getEntityType, dao -> dao)); + daos.forEach(dao -> { + EntityType entityType = dao.getEntityType(); + if (entityType != null) { + this.daos.put(entityType, dao); + } + }); } @SuppressWarnings("unchecked") From 6f9e9305bd6d03a64cda9b54befd74f36446e485 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 15 Apr 2024 16:02:37 +0300 Subject: [PATCH 36/72] Housekeeper refactoring --- .../HousekeeperReprocessingService.java | 3 -- .../housekeeper/HousekeeperService.java | 24 ++++++------ .../stats/HousekeeperStatsService.java | 1 + .../server/controller/AbstractWebTest.java | 13 +++++-- .../housekeeper/HousekeeperServiceTest.java | 37 ++++++------------- .../AlarmsDeletionHousekeeperTask.java | 5 +++ .../EntitiesDeletionHousekeeperTask.java | 2 +- .../LatestTsDeletionHousekeeperTask.java | 5 +++ .../TsHistoryDeletionHousekeeperTask.java | 5 +++ .../trigger/TaskProcessingFailureTrigger.java | 14 +------ 10 files changed, 49 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index ba762facb1..cdc0d3f880 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -40,7 +40,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; @TbCoreComponent @Service @@ -55,8 +54,6 @@ public class HousekeeperReprocessingService { private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); - protected AtomicInteger cycle = new AtomicInteger(); - public HousekeeperReprocessingService(HousekeeperConfig config, @Lazy HousekeeperService housekeeperService, TbCoreQueueFactory queueFactory) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index b16c1b0de9..1f5ed1be58 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; -import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -118,15 +117,19 @@ public class HousekeeperService { throw new IllegalArgumentException("Unsupported task type " + taskType); } - if (log.isDebugEnabled()) { - log.debug("[{}] {} {}", task.getTenantId(), isNew(msg.getTask()) ? "Processing" : "Reprocessing", task.getDescription()); - } + try { + long startTs = System.currentTimeMillis(); Future future = taskExecutor.submit(() -> { taskProcessor.process((T) task); return null; }); future.get(config.getTaskProcessingTimeout(), TimeUnit.MILLISECONDS); + + long timing = System.currentTimeMillis() - startTs; + if (log.isDebugEnabled()) { + log.debug("[{}] Processed {} in {} ms (attempt {})", task.getTenantId(), task.getDescription(), timing, msg.getTask().getAttempt()); + } statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg)); } catch (InterruptedException e) { throw e; @@ -137,14 +140,13 @@ public class HousekeeperService { } else if (e instanceof TimeoutException) { error = new TimeoutException("Timeout after " + config.getTaskProcessingTimeout() + " seconds"); } - log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}", - task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), - taskType, msg.getTask().getAttempt(), task, error); if (msg.getTask().getAttempt() < config.getMaxReprocessingAttempts()) { + log.warn("[{}] Failed to process {} (attempt {}), submitting for reprocessing", + task.getTenantId(), task.getDescription(), msg.getTask().getAttempt(), error); reprocessingService.submitForReprocessing(msg, error); } else { - log.error("Failed to process task in {} attempts: {}", msg.getTask().getAttempt(), msg); + log.error("[{}] Failed to process task in {} attempts: {}", task.getTenantId(), msg.getTask().getAttempt(), msg, e); notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() .task(task) .error(error) @@ -155,12 +157,8 @@ public class HousekeeperService { } } - private boolean isNew(HousekeeperTaskProto task) { - return task.getErrorsCount() == 0; - } - @PreDestroy - private void stop() throws Exception { + private void stop() { consumer.stop(); consumerExecutor.shutdownNow(); log.info("Stopped Housekeeper service"); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 4944cf9f43..da586aa820 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -69,6 +69,7 @@ public class HousekeeperStatsService { } public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + // todo: report timings HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getProcessedCounter().increment(); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 5dd9e4b77f..ee239d12db 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -70,6 +70,7 @@ import org.thingsboard.server.actors.device.DeviceActorMessageProcessor; import org.thingsboard.server.actors.device.SessionInfo; import org.thingsboard.server.actors.device.ToDeviceRpcRequestMetadata; import org.thingsboard.server.actors.service.DefaultActorService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -102,7 +103,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -116,6 +117,7 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.ClaimDevicesService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -231,7 +233,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { private TbTenantProfileService tbTenantProfileService; @Autowired - public TimeseriesService tsService; + protected TimeseriesService tsService; + + @Autowired + protected AttributesService attributesService; @Autowired protected DefaultActorService actorService; @@ -379,7 +384,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } } Awaitility.await("tenant cleanup finish").atMost(30, TimeUnit.SECONDS) - .until(() -> tsService.findLatest(TenantId.SYS_TENANT_ID, tenantId, "test").get().isEmpty()); + .until(() -> attributesService.find(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, "test").get().isEmpty()); } private List getAllTenants() throws Exception { @@ -444,7 +449,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected Tenant createTenant(Tenant tenant) throws Exception { tenant = doPost("/api/tenant", tenant, Tenant.class); - tsService.save(TenantId.SYS_TENANT_ID, tenant.getId(), new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker ts to later know when Housekeeper finishes tenant cleanup + attributesService.save(TenantId.SYS_TENANT_ID, tenant.getId(), AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker attr to later know when Housekeeper finishes tenant cleanup return tenant; } diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index a1015c4a4c..e66d3ed86a 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; @@ -64,7 +63,6 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; @@ -113,8 +111,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { @SpyBean private HousekeeperService housekeeperService; @SpyBean - private HousekeeperClient housekeeperClient; - @SpyBean private HousekeeperReprocessingService housekeeperReprocessingService; @Autowired private EventService eventService; @@ -236,14 +232,17 @@ public class HousekeeperServiceTest extends AbstractControllerTest { List devices = new ArrayList<>(); for (int i = 1; i <= 300; i++) { Device device = createDevice("test" + i, "test" + i); - createRelatedData(device.getId()); devices.add(device.getId()); } + DeviceId firstDevice = devices.get(0); + createRelatedData(firstDevice); + DeviceId lastDevice = devices.get(devices.size() - 1); + createRelatedData(lastDevice); Asset asset = createAsset(); createRelatedData(asset.getId()); - createRelation(devices.get(0), asset.getId()); - createAlarm(devices.get(0), asset.getId()); + createRelation(firstDevice, asset.getId()); + createAlarm(firstDevice, asset.getId()); RuleChainMetaData ruleChainMetaData = createRuleChain(); RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); @@ -261,7 +260,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { loginSysAdmin(); deleteDifferentTenant(); - await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + await().atMost(60, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { for (DeviceId deviceId : devices) { verifyNoRelatedData(deviceId); } @@ -277,7 +276,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } @Test - @Ignore // FIXME !!! public void whenTaskProcessingFails_thenReprocess() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); @@ -289,12 +287,10 @@ public class HousekeeperServiceTest extends AbstractControllerTest { int attempts = 2; await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, 0); - for (int i = 1; i <= attempts; i++) { + for (int i = 0; i <= attempts; i++) { int attempt = i; verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, - task -> task.getErrorsCount() > 0 && task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, attempt); + task -> task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); } }); @@ -306,8 +302,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } @Test - @Ignore // FIXME !!! - public void whenReprocessingAttemptsExceeded_thenReprocessOnNextStartUp() throws Exception { + public void whenReprocessingAttemptsExceeded_thenDropTheTask() throws Exception { TimeoutException error = new TimeoutException("Test timeout"); doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); @@ -327,20 +322,10 @@ public class HousekeeperServiceTest extends AbstractControllerTest { doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); TimeUnit.SECONDS.sleep(2); verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, null))); - - housekeeperReprocessingService.cycle.set(0); // imitating start-up - await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { - verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, 6); - }); } private void verifyTaskProcessing(EntityId entityId, HousekeeperTaskType taskType, int expectedAttempt) throws Exception { - try { - verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); - } catch (Throwable e) { - e.printStackTrace(); - throw e; - } + verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); } private ArgumentMatcher getTaskMatcher(EntityId entityId, HousekeeperTaskType taskType, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java index 9bd8f29bb4..60a50916ef 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -43,4 +43,9 @@ public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { this.alarms = alarms; } + @Override + public String getDescription() { + return super.getDescription() + (alarms != null ? " (" + alarms + ")" : ""); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index e29c3d3c51..d90fa6de94 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -49,7 +49,7 @@ public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { @JsonIgnore @Override public String getDescription() { - return entityType.getNormalName().toLowerCase() + "s deletion"; + return entityType.getNormalName().toLowerCase() + "s deletion" + (entities != null ? " (" + entities + ")" : ""); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java index cef43756e0..4f0f56479e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -36,4 +36,9 @@ public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { this.key = key; } + @Override + public String getDescription() { + return super.getDescription() + (key != null ? " for key '" + key + "'" : ""); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java index 80b8edc345..8f462f2dff 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -36,4 +36,9 @@ public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { this.key = key; } + @Override + public String getDescription() { + return super.getDescription() + (key != null ? " for key '" + key + "'" : ""); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java index 9ccfac6aff..3626e09c0b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java @@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; -import java.util.concurrent.TimeUnit; - @Data @Builder public class TaskProcessingFailureTrigger implements NotificationRuleTrigger { @@ -49,17 +47,7 @@ public class TaskProcessingFailureTrigger implements NotificationRuleTrigger { @Override public boolean deduplicate() { - return true; - } - - @Override - public String getDeduplicationKey() { - return String.join(":", NotificationRuleTrigger.super.getDeduplicationKey(), task.getTaskType().toString()); - } - - @Override - public long getDefaultDeduplicationDuration() { - return TimeUnit.MINUTES.toMillis(30); + return false; } } From be045bcf4251541379efc1135b6b117e92cbb929 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 16 Apr 2024 09:20:59 +0300 Subject: [PATCH 37/72] Updated due to comments --- .../server/common/data/DataConstants.java | 2 ++ .../AbstractGatewaySessionHandler.java | 6 ++---- .../session/GatewaySessionHandlerTest.java | 19 +++++++++++-------- .../service/DefaultTransportService.java | 8 +++++--- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 53b5d2ec63..b529edeb0c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -44,6 +44,8 @@ public class DataConstants { public static final String EDGE_ID = "edgeId"; public static final String DEVICE_ID = "deviceId"; public static final String GATEWAY_PARAMETER = "gateway"; + + public static final String OVERWRITE_ACTIVITY_TIME_PARAMETER = "overwriteActivityTime"; public static final String COAP_TRANSPORT_NAME = "COAP"; public static final String LWM2M_TRANSPORT_NAME = "LWM2M"; public static final String MQTT_TRANSPORT_NAME = "MQTT"; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index b881291756..24a9b665ae 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -95,8 +95,6 @@ public abstract class AbstractGatewaySessionHandler deviceProfileOpt) { log.trace("[{}][{}] onDeviceUpdate: [{}]", gateway.getTenantId(), gateway.getDeviceId(), device); JsonNode deviceAdditionalInfo = device.getAdditionalInfo(); - if (deviceAdditionalInfo.has(GATEWAY_PROPERTY) && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME)) { - overwriteDevicesActivity = deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).asBoolean(); + if (deviceAdditionalInfo.has(DataConstants.GATEWAY_PARAMETER) && deviceAdditionalInfo.has(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER)) { + overwriteDevicesActivity = deviceAdditionalInfo.get(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER).asBoolean(); } } diff --git a/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java b/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java index 9b52d57dc2..158acc2ce2 100644 --- a/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java +++ b/common/transport/mqtt/src/test/java/org/thingsboard/server/transport/mqtt/session/GatewaySessionHandlerTest.java @@ -17,8 +17,10 @@ package org.thingsboard.server.transport.mqtt.session; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -34,6 +36,7 @@ import java.util.concurrent.ConcurrentHashMap; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -41,6 +44,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doNothing; +@ExtendWith(MockitoExtension.class) public class GatewaySessionHandlerTest { @Mock @@ -56,13 +60,12 @@ public class GatewaySessionHandlerTest { @BeforeEach public void setup() { - MockitoAnnotations.openMocks(this); - when(deviceSessionCtx.getSessionId()).thenReturn(UUID.randomUUID()); - doNothing().when(transportService).recordActivity(any()); - when(transportContext.getTransportService()).thenReturn(transportService); - when(deviceSessionCtx.getContext()).thenReturn(transportContext); + lenient().when(deviceSessionCtx.getSessionId()).thenReturn(UUID.randomUUID()); + lenient().doNothing().when(transportService).recordActivity(any()); + lenient().when(transportContext.getTransportService()).thenReturn(transportService); + lenient().when(deviceSessionCtx.getContext()).thenReturn(transportContext); handler = new GatewaySessionHandler(deviceSessionCtx, UUID.randomUUID(), true); - when(handler.getNodeId()).thenReturn("nodeId"); + lenient().when(handler.getNodeId()).thenReturn("nodeId"); } @Test @@ -77,10 +80,10 @@ public class GatewaySessionHandlerTest { deviceInfo.setDeviceType("default"); deviceInfo.setDeviceProfileId(new DeviceProfileId(UUID.randomUUID())); deviceInfo.setAdditionalInfo("{\"gateway\": true, \"overwriteDeviceActivity\": true}"); - when(deviceSessionCtx.getDeviceInfo()).thenReturn(deviceInfo); + lenient().when(deviceSessionCtx.getDeviceInfo()).thenReturn(deviceInfo); GatewayDeviceSessionContext gatewayDeviceSessionContext = new GatewayDeviceSessionContext(handler, deviceInfo, null, null, transportService); devices.put("device1", gatewayDeviceSessionContext); - when(handler.getNodeId()).thenReturn("nodeId"); + lenient().when(handler.getNodeId()).thenReturn("nodeId"); Field devicesField = AbstractGatewaySessionHandler.class.getDeclaredField("devices"); devicesField.setAccessible(true); devicesField.set(handler, devices); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 2f822ccb1a..a756e7a992 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -133,7 +133,6 @@ import java.util.stream.Collectors; @TbTransportComponent public class DefaultTransportService extends TransportActivityManager implements TransportService { - public static final String OVERWRITE_ACTIVITY_TIME = "overwriteActivityTime"; public static final TransportProtos.SessionEventMsg SESSION_EVENT_MSG_OPEN = TransportProtos.SessionEventMsg.newBuilder() .setSessionType(TransportProtos.SessionType.ASYNC) .setEvent(TransportProtos.SessionEvent.OPEN).build(); @@ -1049,8 +1048,11 @@ public class DefaultTransportService extends TransportActivityManager implements .setDeviceName(device.getName()) .setDeviceType(device.getType()).build(); JsonNode deviceAdditionalInfo = device.getAdditionalInfo(); - if (deviceAdditionalInfo.has("gateway") && deviceAdditionalInfo.has(OVERWRITE_ACTIVITY_TIME)) { - md.setOverwriteActivityTime(deviceAdditionalInfo.get(OVERWRITE_ACTIVITY_TIME).asBoolean()); + if (deviceAdditionalInfo.has(DataConstants.GATEWAY_PARAMETER) + && deviceAdditionalInfo.get(DataConstants.GATEWAY_PARAMETER).asBoolean() + && deviceAdditionalInfo.has(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER) + && deviceAdditionalInfo.get(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER).isBoolean()) { + md.setOverwriteActivityTime(deviceAdditionalInfo.get(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER).asBoolean()); } md.setSessionInfo(newSessionInfo); transportCallbackExecutor.submit(() -> md.getListener().onDeviceUpdate(newSessionInfo, device, Optional.ofNullable(newDeviceProfile))); From d8515383c044dc7265982a33ed875376d08cdfa1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Tue, 16 Apr 2024 10:07:41 +0300 Subject: [PATCH 38/72] Updated MqttTransportHandler --- .../server/transport/mqtt/MqttTransportHandler.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 2ecea63e42..b48cb39ca0 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -1083,11 +1083,12 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement try { JsonNode infoNode = context.getMapper().readTree(device.getAdditionalInfo()); if (infoNode != null) { - JsonNode gatewayNode = infoNode.get("gateway"); + JsonNode gatewayNode = infoNode.get(DataConstants.GATEWAY_PARAMETER); if (gatewayNode != null && gatewayNode.asBoolean()) { boolean overwriteDevicesActivity = false; - if (infoNode.has(DefaultTransportService.OVERWRITE_ACTIVITY_TIME) && infoNode.get(DefaultTransportService.OVERWRITE_ACTIVITY_TIME).isBoolean()) { - overwriteDevicesActivity = infoNode.get(DefaultTransportService.OVERWRITE_ACTIVITY_TIME).asBoolean(); + if (infoNode.has(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER) + && infoNode.get(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER).isBoolean()) { + overwriteDevicesActivity = infoNode.get(DataConstants.OVERWRITE_ACTIVITY_TIME_PARAMETER).asBoolean(); sessionMetaData.setOverwriteActivityTime(overwriteDevicesActivity); } gatewaySessionHandler = new GatewaySessionHandler(deviceSessionCtx, sessionId, overwriteDevicesActivity); From a80d4650ea6d4bfdef531be167a7f573055982f6 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 16 Apr 2024 13:00:17 +0300 Subject: [PATCH 39/72] Delete customers last on tenant deletion; tests fixes --- .../controller/AlarmControllerTest.java | 74 +++++++++---------- .../controller/AssetControllerTest.java | 14 +++- .../server/dao/tenant/TenantServiceImpl.java | 17 ++--- 3 files changed, 51 insertions(+), 54 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 5290b429c0..3d0b0f6ea6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -53,9 +53,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.stream.Collectors; -import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -584,7 +583,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - beforeAssignmentTs = System.currentTimeMillis(); + long afterAssignmentTs = System.currentTimeMillis(); loginSysAdmin(); @@ -592,12 +591,12 @@ public class AlarmControllerTest extends AbstractControllerTest { loginDifferentTenant(); - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test @@ -631,16 +630,16 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - beforeAssignmentTs = System.currentTimeMillis(); + long afterAssignmentTs = System.currentTimeMillis(); doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk()); - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test @@ -684,18 +683,18 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - beforeAssignmentTs = System.currentTimeMillis(); + long afterAssignmentTs = System.currentTimeMillis(); Mockito.reset(tbClusterService, auditLogService); doDelete("/api/customer/" + differentTenantCustomerId.getId()).andExpect(status().isOk()); - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test @@ -1003,27 +1002,20 @@ public class AlarmControllerTest extends AbstractControllerTest { types.remove(0); } - foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() { - }) - .getData() - .stream() - .map(EntitySubtype::getType) - .collect(Collectors.toList()); - - Assert.assertEquals(types.size(), foundTypes.size()); - Assert.assertEquals(types, foundTypes); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() {}) + .getData().stream().map(EntitySubtype::getType).toList(); + Assert.assertEquals(types.size(), actualTypes.size()); + Assert.assertEquals(types, actualTypes); + }); doDelete("/api/device/" + devices.get(0).getId()).andExpect(status().isOk()); - foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() { - }) - .getData() - .stream() - .map(EntitySubtype::getType) - .sorted() - .collect(Collectors.toList()); - - Assert.assertTrue(foundTypes.isEmpty()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() {}) + .getData().stream().map(EntitySubtype::getType).sorted().toList(); + Assert.assertTrue(actualTypes.isEmpty()); + }); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java index 06d15ddb44..a250b95082 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.page.PageData; @@ -55,8 +56,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -312,8 +315,9 @@ public class AssetControllerTest extends AbstractControllerTest { alarm = doPost("/api/alarm", alarm, Alarm.class); Assert.assertNotNull(alarm); + AlarmId alarmId = alarm.getId(); - AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarm.getId(), AlarmInfo.class); + AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId, AlarmInfo.class); Assert.assertNotNull(foundAlarm); doDelete("/api/asset/" + savedAsset.getId().getId().toString()) @@ -324,9 +328,11 @@ public class AssetControllerTest extends AbstractControllerTest { .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Asset", assetIdStr)))); - doGet("/api/alarm/info/" + alarm.getId()) - .andExpect(status().isNotFound()) - .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarm.getId().getId().toString())))); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + doGet("/api/alarm/info/" + alarmId) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarmId.getId().toString())))); + }); } @Test diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index da96dee67b..97411e82f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -150,7 +150,6 @@ public class TenantServiceImpl extends AbstractCachedEntityService Date: Tue, 16 Apr 2024 13:01:04 +0300 Subject: [PATCH 40/72] Single entity deletion event when tenant is deleted --- .../service/edge/EdgeEventSourcingListener.java | 14 +++++++++++--- .../entitiy/EntityStateSourcingListener.java | 6 ++++++ .../server/dao/edge/EdgeServiceImpl.java | 1 - 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index d975217e1f..32db644275 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.OAuth2Info; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -44,6 +45,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserServiceImpl; import javax.annotation.PostConstruct; @@ -69,6 +71,7 @@ public class EdgeEventSourcingListener { private final TbClusterService tbClusterService; private final EdgeSynchronizationManager edgeSynchronizationManager; + private final TenantService tenantService; @PostConstruct public void init() { @@ -95,19 +98,24 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); EntityType entityType = event.getEntityId().getEntityType(); + if (!tenantService.tenantExists(tenantId)) { + log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); + return; + } try { if (EntityType.EDGE.equals(entityType) || EntityType.TENANT.equals(entityType)) { return; } - log.trace("[{}] DeleteEntityEvent called: {}", event.getTenantId(), event); + log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event); EdgeEventType type = getEdgeEventTypeForEntityEvent(event.getEntity()); EdgeEventActionType actionType = getEdgeEventActionTypeForEntityEvent(event.getEntity()); - tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(), + tbClusterService.sendNotificationMsgToEdge(tenantId, null, event.getEntityId(), JacksonUtil.toString(event.getEntity()), type, actionType, edgeSynchronizationManager.getEdgeId().get()); } catch (Exception e) { - log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e); + log.error("[{}] failed to process DeleteEntityEvent: {}", tenantId, event, e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index bc7dc62e78..877e252151 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNoti import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.tenant.TenantService; import javax.annotation.PostConstruct; import java.util.Set; @@ -61,6 +62,7 @@ import java.util.Set; public class EntityStateSourcingListener { private final TbClusterService tbClusterService; + private final TenantService tenantService; @PostConstruct public void init() { @@ -121,6 +123,10 @@ public class EntityStateSourcingListener { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); EntityType entityType = entityId.getEntityType(); + if (entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); + return; + } log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 466a591394..66cf74ba9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -231,7 +231,6 @@ public class EdgeServiceImpl extends AbstractCachedEntityService Date: Tue, 16 Apr 2024 14:01:11 +0300 Subject: [PATCH 41/72] Minor refactoring --- application/src/test/resources/logback-test.xml | 1 + .../org/thingsboard/server/queue/TbQueueConsumer.java | 2 -- .../server/common/data/housekeeper/HousekeeperTask.java | 9 --------- .../queue/common/AbstractTbQueueConsumerTemplate.java | 5 ----- .../server/queue/memory/InMemoryTbQueueConsumer.java | 5 ----- .../grafana/provisioning/dashboards/housekeeper.json | 6 +++--- 6 files changed, 4 insertions(+), 24 deletions(-) diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index d72bccb7a6..4651f2f359 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -31,6 +31,7 @@ + diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 3747e6d026..51456ab0e2 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -36,8 +36,6 @@ public interface TbQueueConsumer { void commit(); - boolean isCommitSupported(); - boolean isStopped(); List getFullTopicNames(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 344edc3f0d..8354b2b861 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -60,19 +60,10 @@ public class HousekeeperTask implements Serializable { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); } - /** - * Task for telemetry deletion, processed by TelemetryDeletionTaskProcessor. - * The processor will submit separate tasks for each key for latest and ts history deletion, - * processed by LatestTsDeletionTaskProcessor and TsHistoryDeletionTaskProcessor accordingly. - * */ public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY); } - /** - * Task for events deletion, processed by EventsDeletionTaskProcessor. - * All types of events will be dropped for this entity. - * */ public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) { return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 89840d81ad..ac263c10c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -201,11 +201,6 @@ public abstract class AbstractTbQueueConsumerTemplate i return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } - @Override - public boolean isCommitSupported() { - return true; - } - protected boolean isLongPollingSupported() { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 269924087b..de3833d124 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -114,9 +114,4 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); } - @Override - public boolean isCommitSupported() { - return false; - } - } diff --git a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json index 8c68cc9428..b604b96135 100644 --- a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json +++ b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json @@ -188,7 +188,7 @@ "uid": "9BonzvTSz" }, "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedProcessing\"}[1m]))", + "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"failedProcessing\"}[1m]))", "interval": "", "legendFormat": "{{taskType}}", "refId": "A" @@ -274,7 +274,7 @@ "uid": "9BonzvTSz" }, "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"reprocessed\"}[1m]))", + "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"reprocessed\"}[1m]))", "interval": "", "legendFormat": "{{taskType}}", "refId": "A" @@ -360,7 +360,7 @@ "uid": "9BonzvTSz" }, "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"failedReprocessing\"}[1m]))", + "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"failedReprocessing\"}[1m]))", "interval": "", "legendFormat": "{{taskType}}", "refId": "A" From 9e5c6cf446d8b0c6460c809c0357d35d9c46da47 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 16 Apr 2024 14:16:07 +0300 Subject: [PATCH 42/72] Remove housekeeper.json --- .../provisioning/dashboards/housekeeper.json | 390 ------------------ 1 file changed, 390 deletions(-) delete mode 100644 docker/monitoring/grafana/provisioning/dashboards/housekeeper.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json b/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json deleted file mode 100644 index b604b96135..0000000000 --- a/docker/monitoring/grafana/provisioning/dashboards/housekeeper.json +++ /dev/null @@ -1,390 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 2, - "links": [], - "liveNow": false, - "panels": [ - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "9BonzvTSz" - }, - "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{instance=\"192.168.3.27:8080\",statsName=\"processed\"}[1m]))", - "interval": "", - "legendFormat": "{{taskType}}", - "refId": "A" - } - ], - "title": "Processed", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "9BonzvTSz" - }, - "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"failedProcessing\"}[1m]))", - "interval": "", - "legendFormat": "{{taskType}}", - "refId": "A" - } - ], - "title": "Processing failures", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "9BonzvTSz" - }, - "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"reprocessed\"}[1m]))", - "interval": "", - "legendFormat": "{{taskType}}", - "refId": "A" - } - ], - "title": "Reprocessed", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "9BonzvTSz" - }, - "exemplar": true, - "expr": "sum by (taskType) (increase(housekeeper_total{statsName=\"failedReprocessing\"}[1m]))", - "interval": "", - "legendFormat": "{{taskType}}", - "refId": "A" - } - ], - "title": "Reprocessing failures", - "type": "timeseries" - } - ], - "refresh": "10s", - "schemaVersion": 35, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Housekeeper", - "uid": "JFJb7voIz", - "version": 5, - "weekStart": "" -} \ No newline at end of file From 49a1a37b0669a3261c1385b668e1fe58765226d6 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 17 Apr 2024 13:07:50 +0300 Subject: [PATCH 43/72] Refactor deletion events handling --- .../server/service/edge/EdgeEventSourcingListener.java | 2 +- .../server/service/entitiy/EntityStateSourcingListener.java | 2 +- .../org/thingsboard/server/dao/tenant/TenantServiceImpl.java | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index 32db644275..651339e3a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -100,7 +100,7 @@ public class EdgeEventSourcingListener { public void handleEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityType entityType = event.getEntityId().getEntityType(); - if (!tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 877e252151..c013650fa7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -123,7 +123,7 @@ public class EntityStateSourcingListener { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); EntityType entityType = entityId.getEntityType(); - if (entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); return; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 97411e82f1..83f1a636c9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -160,9 +160,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService Date: Wed, 17 Apr 2024 15:03:02 +0300 Subject: [PATCH 44/72] Processing time stats for Housekeeper --- .../HousekeeperReprocessingService.java | 4 +- .../housekeeper/HousekeeperService.java | 2 +- .../stats/HousekeeperStatsService.java | 18 ++++-- .../common/stats/DefaultStatsFactory.java | 28 +++++++--- .../server/common/stats/StatsFactory.java | 2 + .../server/common/stats/StatsTimer.java | 55 +++++++++++++++++++ 6 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java index cdc0d3f880..ae16f925a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -77,6 +77,8 @@ public class HousekeeperReprocessingService { } private void processMsgs(List> msgs, TbQueueConsumer> consumer) throws Exception { + Thread.sleep(config.getTaskReprocessingDelay()); + for (TbProtoQueueMsg msg : msgs) { log.trace("Reprocessing task: {}", msg); try { @@ -89,8 +91,6 @@ public class HousekeeperReprocessingService { } } consumer.commit(); - - Thread.sleep(config.getTaskReprocessingDelay()); } public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 1f5ed1be58..898ae74d3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -130,7 +130,7 @@ public class HousekeeperService { if (log.isDebugEnabled()) { log.debug("[{}] Processed {} in {} ms (attempt {})", task.getTenantId(), task.getDescription(), timing, msg.getTask().getAttempt()); } - statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg)); + statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg, timing)); } catch (InterruptedException e) { throw e; } catch (Throwable e) { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index da586aa820..6723d51625 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsTimer; import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; @@ -53,14 +54,14 @@ public class HousekeeperStatsService { String statsStr = stats.values().stream().map(stats -> { String countersStr = stats.getCounters().stream() .filter(counter -> counter.get() > 0) - .map(counter -> counter.getName() + " = " + counter.get()) - .collect(Collectors.joining(", ")); + .map(counter -> counter.getName() + " = [" + counter.get() + "]") + .collect(Collectors.joining(" ")); if (countersStr.isEmpty()) { return null; } else { - return stats.getTaskType() + " {" + countersStr + "}"; + return stats.getTaskType() + " " + countersStr + " avgProcessingTime [" + stats.getProcessingTimer().getAvg() + " ms]"; } - }).filter(Objects::nonNull).collect(Collectors.joining("; ")); + }).filter(Objects::nonNull).collect(Collectors.joining(", ")); if (!statsStr.isEmpty()) { stats.values().forEach(HousekeeperStats::reset); @@ -68,14 +69,14 @@ public class HousekeeperStatsService { } } - public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { - // todo: report timings + public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg, long timing) { HousekeeperStats stats = this.stats.get(taskType); if (msg.getTask().getErrorsCount() == 0) { stats.getProcessedCounter().increment(); } else { stats.getReprocessedCounter().increment(); } + stats.getProcessingTimer().record(timing); } public void reportFailure(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { @@ -97,12 +98,15 @@ public class HousekeeperStatsService { private final StatsCounter reprocessedCounter; private final StatsCounter failedReprocessingCounter; + private final StatsTimer processingTimer; + public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) { this.taskType = taskType; this.processedCounter = register("processed", statsFactory); this.failedProcessingCounter = register("failedProcessing", statsFactory); this.reprocessedCounter = register("reprocessed", statsFactory); this.failedReprocessingCounter = register("failedReprocessing", statsFactory); + this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name()); } private StatsCounter register(String statsName, StatsFactory statsFactory) { @@ -113,7 +117,9 @@ public class HousekeeperStatsService { public void reset() { counters.forEach(DefaultCounter::clear); + processingTimer.reset(); } + } } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index be0c4fe19f..ca97792186 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -19,13 +19,13 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; +import jakarta.annotation.PostConstruct; import org.apache.commons.lang3.ArrayUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; -import jakarta.annotation.PostConstruct; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -63,13 +63,7 @@ public class DefaultStatsFactory implements StatsFactory { @Override public StatsCounter createStatsCounter(String key, String statsName, String... otherTags) { - String[] tags = new String[]{STATS_NAME_TAG, statsName}; - if (otherTags.length > 0) { - if (otherTags.length % 2 != 0) { - throw new IllegalArgumentException("Invalid tags array size"); - } - tags = ArrayUtils.addAll(tags, otherTags); - } + String[] tags = getTags(statsName, otherTags); return new StatsCounter( new AtomicInteger(0), metricsEnabled ? meterRegistry.counter(key, tags) : STUB_COUNTER, @@ -111,6 +105,24 @@ public class DefaultStatsFactory implements StatsFactory { return timerBuilder.register(meterRegistry); } + @Override + public StatsTimer createTimer(StatsType type, String name, String... tags) { + return new StatsTimer(name, Timer.builder(type.getName()) + .tags(getTags(name, tags)) + .register(meterRegistry)); + } + + private static String[] getTags(String statsName, String[] otherTags) { + String[] tags = new String[]{STATS_NAME_TAG, statsName}; + if (otherTags.length > 0) { + if (otherTags.length % 2 != 0) { + throw new IllegalArgumentException("Invalid tags array size"); + } + tags = ArrayUtils.addAll(tags, otherTags); + } + return tags; + } + private static class StubCounter implements Counter { @Override public void increment(double amount) { diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index b9bf0b97fb..a6cc4aa1d4 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -29,4 +29,6 @@ public interface StatsFactory { Timer createTimer(String key, String... tags); + StatsTimer createTimer(StatsType type, String name, String... tags); + } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java new file mode 100644 index 0000000000..ca74444619 --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +import io.micrometer.core.instrument.Timer; +import lombok.Getter; + +import java.util.concurrent.TimeUnit; + +public class StatsTimer { + + @Getter + private final String name; + private final Timer timer; + + private int count; + private long totalTime; + + public StatsTimer(String name, Timer micrometerTimer) { + this.name = name; + this.timer = micrometerTimer; + } + + public void record(long timeMs) { + count++; + totalTime += timeMs; + timer.record(timeMs, TimeUnit.MILLISECONDS); + } + + public double getAvg() { + if (count == 0) { + return 0.0; + } + return (double) totalTime / count; + } + + public void reset() { + count = 0; + totalTime = 0; + } + +} From 3803081b578355ffe479e62258bdcdf10725a9a5 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 18 Apr 2024 13:25:40 +0300 Subject: [PATCH 45/72] Refactor alarms deletion; separate tenant entities deletion task --- .../AlarmsDeletionTaskProcessor.java | 16 ++--- .../EntitiesDeletionTaskProcessor.java | 27 ++------ .../TenantEntitiesDeletionTaskProcessor.java | 63 +++++++++++++++++++ .../housekeeper/HousekeeperServiceTest.java | 6 +- .../server/dao/alarm/AlarmService.java | 4 +- .../EntitiesDeletionHousekeeperTask.java | 9 +-- .../data/housekeeper/HousekeeperTask.java | 5 +- .../data/housekeeper/HousekeeperTaskType.java | 5 +- ...TenantEntitiesDeletionHousekeeperTask.java | 45 +++++++++++++ .../server/dao/alarm/AlarmDao.java | 3 +- .../server/dao/alarm/BaseAlarmService.java | 6 +- .../dao/housekeeper/CleanUpService.java | 2 +- .../server/dao/sql/alarm/AlarmRepository.java | 21 +++++-- .../server/dao/sql/alarm/JpaAlarmDao.java | 17 ++++- .../dao/service/AbstractServiceTest.java | 6 +- 15 files changed, 174 insertions(+), 61 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java index 09fa0977d5..cc6cb9bbab 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -24,12 +24,11 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.alarm.AlarmService; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @Component @RequiredArgsConstructor @@ -46,16 +45,19 @@ public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor alarms = alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, entityId, last, 128); + List> alarms = alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, lastCreatedTime, lastId, 128); if (alarms.isEmpty()) { break; } - housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream() - .map(UUIDBased::getId).collect(Collectors.toList()))); - last = alarms.get(alarms.size() - 1); + housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream().map(TbPair::getFirst).toList())); + + TbPair last = alarms.get(alarms.size() - 1); + lastId = new AlarmId(last.getFirst()); + lastCreatedTime = last.getSecond(); log.debug("[{}][{}][{}] Submitted task for deleting {} alarms", tenantId, entityType, entityId, alarms.size()); } } else { diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java index 00c8a7cc61..186a290528 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -24,12 +24,9 @@ import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.entity.EntityDaoRegistry; import org.thingsboard.server.dao.entity.EntityDaoService; import org.thingsboard.server.dao.entity.EntityServiceRegistry; -import java.util.List; import java.util.UUID; @Component @@ -37,35 +34,19 @@ import java.util.UUID; @Slf4j public class EntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { - private final EntityDaoRegistry entityDaoRegistry; private final EntityServiceRegistry entityServiceRegistry; @Override public void process(EntitiesDeletionHousekeeperTask task) throws Exception { EntityType entityType = task.getEntityType(); TenantId tenantId = task.getTenantId(); - Dao entityDao = entityDaoRegistry.getDao(entityType); EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); - if (task.getEntities() == null) { - UUID last = null; - while (true) { - List entities = entityDao.findIdsByTenantIdAndIdOffset(tenantId, last, 128); - if (entities.isEmpty()) { - break; - } - - housekeeperClient.submitTask(new EntitiesDeletionHousekeeperTask(tenantId, entityType, entities)); - last = entities.get(entities.size() - 1); - log.debug("[{}] Submitted task for deleting {} {}s", tenantId, entities.size(), entityType.getNormalName().toLowerCase()); - } - } else { - for (UUID entityUuid : task.getEntities()) { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - entityService.deleteEntity(tenantId, entityId, true); - } - log.debug("[{}] Deleted {} {}s", tenantId, task.getEntities().size(), entityType.getNormalName().toLowerCase()); + for (UUID entityUuid : task.getEntities()) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + entityService.deleteEntity(tenantId, entityId, true); } + log.debug("[{}] Deleted {} {}s", tenantId, task.getEntities().size(), entityType.getNormalName().toLowerCase()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java new file mode 100644 index 0000000000..386afb1517 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.TenantEntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; + +import java.util.List; +import java.util.UUID; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TenantEntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final EntityDaoRegistry entityDaoRegistry; + + @Override + public void process(TenantEntitiesDeletionHousekeeperTask task) throws Exception { + EntityType entityType = task.getEntityType(); + TenantId tenantId = task.getTenantId(); + Dao entityDao = entityDaoRegistry.getDao(entityType); + + UUID last = null; + while (true) { + List entities = entityDao.findIdsByTenantIdAndIdOffset(tenantId, last, 128); + if (entities.isEmpty()) { + break; + } + + housekeeperClient.submitTask(new EntitiesDeletionHousekeeperTask(tenantId, entityType, entities)); + last = entities.get(entities.size() - 1); + log.debug("[{}] Submitted task for deleting {} {}s", tenantId, entities.size(), entityType.getNormalName().toLowerCase()); + } + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_TENANT_ENTITIES; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index e66d3ed86a..e8f1a43ad1 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -360,7 +360,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { } private void verifyNoAlarms(EntityId entityId) { - assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, entityId, null, 10)).isEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, 0, null, 10)).isEmpty(); } private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception { @@ -402,7 +402,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { List entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId()); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType())); assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType())); - assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, deviceId, null, 10)).isNotEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty(); } private void createAlarm(DeviceId deviceId) { @@ -412,7 +412,7 @@ public class HousekeeperServiceTest extends AbstractControllerTest { .severity(AlarmSeverity.CRITICAL) .type("test alarm for " + deviceId + " " + RandomStringUtils.randomAlphabetic(10)) .build(), Alarm.class); - assertThat(alarmService.findAlarmIdsByOriginatorIdAndIdOffset(tenantId, deviceId, null, 10)).isNotEmpty(); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty(); } private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index ee584ff856..5bf6b95853 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -38,11 +38,13 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.Collection; import java.util.List; import java.util.Set; +import java.util.UUID; public interface AlarmService extends EntityDaoService { @@ -107,7 +109,7 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - List findAlarmIdsByOriginatorIdAndIdOffset(TenantId tenantId, EntityId originatorId, AlarmId idOffset, int limit); + List> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index d90fa6de94..b98af48d42 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -16,7 +16,6 @@ package org.thingsboard.server.common.data.housekeeper; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.AccessLevel; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; @@ -30,16 +29,12 @@ import java.util.UUID; @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) -@NoArgsConstructor(access = AccessLevel.PROTECTED) +@NoArgsConstructor public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { private EntityType entityType; private List entities; - protected EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { - this(tenantId, entityType, null); - } - public EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType, List entities) { super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); this.entityType = entityType; @@ -49,7 +44,7 @@ public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { @JsonIgnore @Override public String getDescription() { - return entityType.getNormalName().toLowerCase() + "s deletion" + (entities != null ? " (" + entities + ")" : ""); + return entityType.getNormalName().toLowerCase() + "s deletion (" + entities + ")"; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 8354b2b861..8757b1121e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -36,6 +36,7 @@ import java.io.Serializable; @JsonSubTypes({ @Type(name = "DELETE_TS_HISTORY", value = TsHistoryDeletionHousekeeperTask.class), @Type(name = "DELETE_LATEST_TS", value = LatestTsDeletionHousekeeperTask.class), + @Type(name = "DELETE_TENANT_ENTITIES", value = TenantEntitiesDeletionHousekeeperTask.class), @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), @Type(name = "DELETE_ALARMS", value = AlarmsDeletionHousekeeperTask.class), @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) @@ -76,8 +77,8 @@ public class HousekeeperTask implements Serializable { return new AlarmsDeletionHousekeeperTask(tenantId, entityId); } - public static HousekeeperTask deleteEntities(TenantId tenantId, EntityType entityType) { - return new EntitiesDeletionHousekeeperTask(tenantId, entityType); + public static HousekeeperTask deleteTenantEntities(TenantId tenantId, EntityType entityType) { + return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); } @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java index 572e85f2cf..edd13b3d5d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -22,14 +22,15 @@ import lombok.RequiredArgsConstructor; @Getter public enum HousekeeperTaskType { - DELETE_ENTITIES("entities deletion"), DELETE_ATTRIBUTES("attributes deletion"), DELETE_TELEMETRY("telemetry deletion"), DELETE_LATEST_TS("latest telemetry deletion"), DELETE_TS_HISTORY("timeseries history deletion"), DELETE_EVENTS("events deletion"), DELETE_ALARMS("alarms deletion"), - UNASSIGN_ALARMS("alarms unassigning"); + UNASSIGN_ALARMS("alarms unassigning"), + DELETE_TENANT_ENTITIES("tenant entities deletion"), + DELETE_ENTITIES("entities deletion"); private final String description; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java new file mode 100644 index 0000000000..f6c6d5030c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask { + + private EntityType entityType; + + public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { + super(tenantId, tenantId, HousekeeperTaskType.DELETE_TENANT_ENTITIES); + this.entityType = entityType; + } + + @JsonIgnore + @Override + public String getDescription() { + return entityType.getNormalName().toLowerCase() + "s deletion"; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 03038b3bfe..570256e98c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.Dao; import java.util.Collection; @@ -80,7 +81,7 @@ public interface AlarmDao extends Dao { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UUID userId, PageLink pageLink); - PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, AlarmId idOffset, PageLink pageLink); + PageData> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); void createEntityAlarmRecord(EntityAlarm entityAlarm); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index 9d0940a605..f0fc18c827 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; @@ -75,6 +76,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -313,9 +315,9 @@ public class BaseAlarmService extends AbstractCachedEntityService findAlarmIdsByOriginatorIdAndIdOffset(TenantId tenantId, EntityId originatorId, AlarmId idOffset, int limit) { + public List> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit) { log.trace("[{}] Executing findAlarmIdsByOriginatorIdAndIdOffset [{}][{}]", tenantId, originatorId, idOffset); - return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, idOffset, new PageLink(limit)).getData(); + return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, createdTimeOffset, idOffset, limit).getData(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 48531aab90..082a60f44a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -77,7 +77,7 @@ public class CleanUpService { public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { for (EntityType entityType : entityTypes) { - housekeeperClient.submitTask(HousekeeperTask.deleteEntities(tenantId, entityType)); + housekeeperClient.submitTask(HousekeeperTask.deleteTenantEntities(tenantId, entityType)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index ff9a2ed7d6..7bb16a9798 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -24,6 +24,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.model.sql.AlarmInfoEntity; @@ -334,12 +335,20 @@ public interface AlarmRepository extends JpaRepository { Page findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, @Param("assigneeId") UUID assigneeId, Pageable pageable); // using Slice so that count query is not executed - @Query(value = "SELECT id FROM alarm WHERE tenant_id = :tenantId AND originator_id = :originatorId " + - "AND (cast(:idOffset as uuid) IS NULL OR id > cast(:idOffset as uuid))", nativeQuery = true) - Slice findAlarmIdsByOriginatorId(@Param("tenantId") UUID tenantId, - @Param("originatorId") UUID originatorId, - @Param("idOffset") UUID idOffset, - Pageable pageable); + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.originatorId = :originatorId " + + "AND (a.createdTime > :createdTimeOffset OR " + + "(a.createdTime = :createdTimeOffset AND a.id > :idOffset))") + Slice> findAlarmIdsByOriginatorId(@Param("originatorId") UUID originatorId, + @Param("createdTimeOffset") long createdTimeOffset, + @Param("idOffset") UUID idOffset, + Pageable pageable); + + // using Slice so that count query is not executed + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.originatorId = :originatorId") + Slice> findAlarmIdsByOriginatorId(@Param("originatorId") UUID originatorId, + Pageable pageable); @Query(value = "SELECT create_or_update_active_alarm(:t_id, :c_id, :a_id, :a_created_ts, :a_o_id, :a_o_type, :a_type, :a_severity, " + ":a_start_ts, :a_end_ts, :a_details, :a_propagate, :a_propagate_to_owner, " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index da603a6c6f..c3e6e123f5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -48,9 +50,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; @@ -69,6 +73,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import static org.thingsboard.server.common.data.page.SortOrder.Direction.ASC; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; import static org.thingsboard.server.dao.DaoUtil.toPageable; @@ -297,9 +302,15 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, AlarmId idOffset, PageLink pageLink) { - return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), idOffset != null ? idOffset.getId() : null, DaoUtil.toPageable(pageLink))) - .mapData(AlarmId::new); + public PageData> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit) { + Slice> result; + Pageable pageRequest = toPageable(new PageLink(limit), List.of(SortOrder.of("createdTime", ASC), SortOrder.of("id", ASC))); + if (idOffset == null) { + result = alarmRepository.findAlarmIdsByOriginatorId(originatorId.getId(), pageRequest); + } else { + result = alarmRepository.findAlarmIdsByOriginatorId(originatorId.getId(), createdTimeOffset, idOffset.getId(), pageRequest); + } + return DaoUtil.pageToPageData(result); } @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index f1e1fd0e76..6127a4192a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -42,7 +42,7 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTra import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; -import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.TenantEntitiesDeletionHousekeeperTask; import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -142,8 +142,8 @@ public abstract class AbstractServiceTest { @Bean public HousekeeperClient housekeeperClient() { return task -> { - if (task.getTaskType() == HousekeeperTaskType.DELETE_ENTITIES) { - EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(((EntitiesDeletionHousekeeperTask) task).getEntityType()); + if (task.getTaskType() == HousekeeperTaskType.DELETE_TENANT_ENTITIES) { + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(((TenantEntitiesDeletionHousekeeperTask) task).getEntityType()); entityService.deleteByTenantId(task.getTenantId()); } }; From c06c826329a0bc6a6fccafc3e68d306bb80094df Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 22 Apr 2024 15:07:53 +0300 Subject: [PATCH 46/72] Fix tenant deletion tests --- .../server/service/notification/NotificationApiTest.java | 6 ++++-- .../service/notification/NotificationTargetApiTest.java | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 060af9eac1..f7864a1df7 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -323,8 +323,10 @@ public class NotificationApiTest extends AbstractNotificationApiTest { deleteDifferentTenant(); - assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements()) - .isZero(); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements()) + .isZero(); + }); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java index c7ef9a5836..a5bdc6c2e4 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java @@ -38,8 +38,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -148,7 +150,9 @@ public class NotificationTargetApiTest extends AbstractNotificationApiTest { assertThat(notificationTargetDao.findByTenantIdAndSupportedNotificationTypeAndPageLink(differentTenantId, NotificationType.GENERAL, new PageLink(10)).getData()).isNotEmpty(); deleteDifferentTenant(); - assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty(); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty(); + }); } @Test From baf36bc2ea5033bf8f1ea1984ff83bbb8067b960 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 22 Apr 2024 18:12:20 +0300 Subject: [PATCH 47/72] Refactor CleanUpService --- .../dao/housekeeper/CleanUpService.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java index 082a60f44a..d64e6ea536 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.relation.RelationService; import java.util.EnumSet; +import java.util.Optional; import java.util.Set; @Component @@ -38,7 +39,7 @@ import java.util.Set; @Slf4j public class CleanUpService { - private final HousekeeperClient housekeeperClient; + private final Optional housekeeperClient; private final RelationService relationService; private final Set skippedEntities = EnumSet.of( @@ -59,7 +60,7 @@ public class CleanUpService { cleanUpRelatedData(tenantId, entityId); } if (entityType == EntityType.USER) { - housekeeperClient.submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); } } catch (Throwable e) { log.error("[{}][{}][{}] Failed to handle entity deletion event", tenantId, entityType, entityId.getId(), e); @@ -69,16 +70,22 @@ public class CleanUpService { public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { log.debug("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); relationService.deleteEntityRelations(tenantId, entityId); - housekeeperClient.submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); - housekeeperClient.submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); - housekeeperClient.submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); - housekeeperClient.submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); + submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); } public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { for (EntityType entityType : entityTypes) { - housekeeperClient.submitTask(HousekeeperTask.deleteTenantEntities(tenantId, entityType)); + submitTask(HousekeeperTask.deleteTenantEntities(tenantId, entityType)); } } + private void submitTask(HousekeeperTask task) { + housekeeperClient.ifPresent(housekeeperClient -> { + housekeeperClient.submitTask(task); + }); + } + } From 572b680b29f381a427ccfa483fc6a4dc840633ed Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 24 Apr 2024 12:35:47 +0300 Subject: [PATCH 48/72] Refactor tenant deletion in tests --- .../server/controller/AbstractWebTest.java | 21 +++++-------------- .../controller/AssetControllerTest.java | 6 ++---- .../AssetProfileControllerTest.java | 3 +-- .../controller/AuditLogControllerTest.java | 3 +-- .../ComponentDescriptorControllerTest.java | 3 +-- .../controller/CustomerControllerTest.java | 3 +-- .../controller/DashboardControllerTest.java | 6 ++---- .../DeviceConnectivityControllerTest.java | 3 +-- .../controller/DeviceControllerTest.java | 10 +++------ .../DeviceProfileControllerTest.java | 3 +-- .../server/controller/EdgeControllerTest.java | 3 +-- .../controller/EntityQueryControllerTest.java | 4 +--- .../EntityRelationControllerTest.java | 3 +-- .../controller/EntityViewControllerTest.java | 3 +-- .../server/controller/HomePageApiTest.java | 2 +- .../controller/OtaPackageControllerTest.java | 3 +-- .../server/controller/RpcControllerTest.java | 3 +-- .../controller/RuleChainControllerTest.java | 3 +-- .../controller/TbResourceControllerTest.java | 3 +-- .../controller/TenantControllerTest.java | 14 +++++-------- .../TenantProfileControllerTest.java | 3 +-- .../server/controller/UserControllerTest.java | 3 +-- .../controller/WidgetTypeControllerTest.java | 3 +-- .../WidgetsBundleControllerTest.java | 3 +-- ...AbstractRuleEngineFlowIntegrationTest.java | 2 +- .../sql/BaseTbResourceServiceTest.java | 9 +++----- .../SequentialTimeseriesPersistenceTest.java | 2 +- 27 files changed, 41 insertions(+), 86 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index ee239d12db..75d127a29d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -25,7 +25,6 @@ import io.jsonwebtoken.Header; import io.jsonwebtoken.Jwt; import io.jsonwebtoken.Jwts; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpStatus; import org.awaitility.Awaitility; import org.hamcrest.Matcher; import org.hibernate.exception.ConstraintViolationException; @@ -367,21 +366,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { assertThat(loadedTenants).as("All tenants expected to be deleted, but some tenants left in the database").isEmpty(); } - private void deleteTenant(TenantId tenantId) { - int status = 0; - int retries = 0; - while (status != HttpStatus.SC_OK && retries < CLEANUP_TENANT_RETRIES_COUNT) { - retries++; - try { - status = doDelete("/api/tenant/" + tenantId.getId().toString()) - .andReturn().getResponse().getStatus(); - if (status != HttpStatus.SC_OK) { - log.warn("Tenant deletion failed, tenantId: {}", tenantId.getId().toString()); - Thread.sleep(1000L); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + protected void deleteTenant(TenantId tenantId) { + try { + doDelete("/api/tenant/" + tenantId.getId()).andExpect(status().isOk()); + } catch (Exception e) { + throw new RuntimeException(e); } Awaitility.await("tenant cleanup finish").atMost(30, TimeUnit.SECONDS) .until(() -> attributesService.find(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, "test").get().isEmpty()); diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java index a250b95082..9e0ae0ab4c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java @@ -107,8 +107,7 @@ public class AssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -588,8 +587,7 @@ public class AssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java index 432c309a4e..e563b13179 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java @@ -99,8 +99,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index bd2fe2e3f4..4059c3a70a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -92,8 +92,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java index dd9eea7ea3..4b3d1265ea 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java @@ -64,8 +64,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java index 370ba41422..544096d27e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java @@ -104,8 +104,7 @@ public class CustomerControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java index 71c758c1e1..db7acafdb9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java @@ -94,8 +94,7 @@ public class DashboardControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -321,8 +320,7 @@ public class DashboardControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 8a67fcf396..1bc51e90e2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -187,8 +187,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index e59595e9c7..1cc1468c68 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -151,8 +151,7 @@ public class DeviceControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -721,9 +720,7 @@ public class DeviceControllerTest extends AbstractControllerTest { testNotificationUpdateGatewayNever(); loginSysAdmin(); - - doDelete("/api/tenant/" + savedTenant2.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test @@ -1380,8 +1377,7 @@ public class DeviceControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString(msgErrorNoFound("Device", savedAnotherDevice.getId().getId().toString())))); loginSysAdmin(); - doDelete("/api/tenant/" + savedDifferentTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedDifferentTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java index 4a93e9c0ac..a9d15ac3fc 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java @@ -110,8 +110,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 73a3a2e25e..f8ac2b7dda 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -394,8 +394,7 @@ public class EdgeControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 8de8514e5b..f12dad6e4e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -99,9 +99,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { @After public void afterTest() throws Exception { loginSysAdmin(); - - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index 6c613ada60..7068eafb61 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -93,8 +93,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 3287af19b6..2adf5b37b5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -398,8 +398,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java index 717c17d437..21d6f50748 100644 --- a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java @@ -115,7 +115,7 @@ public class HomePageApiTest extends AbstractControllerTest { Assert.assertEquals(initialCount + 100, update.getCount()); for (Tenant tenant : tenants) { - doDelete("/api/tenant/" + tenant.getId().toString()); + deleteTenant(tenant.getId()); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java index 9b5e0a65f8..33da6f4a80 100644 --- a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java @@ -94,8 +94,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java index 4d1f3d6407..8bc32c2103 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java @@ -66,8 +66,7 @@ public class RpcControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } private Device createDefaultDevice() { diff --git a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java index 7595a69931..28b6c75a12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java @@ -99,8 +99,7 @@ public class RuleChainControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index 1d875d2ccc..fad526bb78 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -90,8 +90,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 63747d2c19..6edcd77694 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -155,8 +155,7 @@ public class TenantControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.UPDATED, 1); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.DELETED, 1); } @@ -185,8 +184,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(savedTenant, foundTenant); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -198,8 +196,7 @@ public class TenantControllerTest extends AbstractControllerTest { TenantInfo foundTenant = doGet("/api/tenant/info/" + savedTenant.getId().getId().toString(), TenantInfo.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(new TenantInfo(savedTenant, "Default"), foundTenant); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -240,8 +237,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); String tenantIdStr = savedTenant.getId().getId().toString(); - doDelete("/api/tenant/" + tenantIdStr) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); doGet("/api/tenant/" + tenantIdStr) .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr)))); @@ -559,7 +555,7 @@ public class TenantControllerTest extends AbstractControllerTest { } loginSysAdmin(); - doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(tenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index f25cfe2228..15384a1ed0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -186,8 +186,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventNeverTenantProfile(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java index 72930e0327..7ad2e1771b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java @@ -422,8 +422,7 @@ public class UserControllerTest extends AbstractControllerTest { assertThat(tenantAdmins).as("admins list size").hasSameSizeAs(loadedTenantAdmins); assertThat(tenantAdmins).as("admins list content").isEqualTo(loadedTenantAdmins); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); pageLink = new PageLink(33); pageData = doGetTypedWithPageLink("/api/tenant/" + tenantId.getId().toString() + "/users?", diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index c689b12b43..19e05fbd79 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -70,8 +70,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java index bb7bab22f3..6ff9bf2c6e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java @@ -70,8 +70,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 1a172be8dd..7238ae8bf6 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -121,7 +121,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule public void afterTest() throws Exception { loginSysAdmin(); if (savedTenant != null) { - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } } diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index d9b33edba2..34f43512bf 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -147,8 +147,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -441,8 +440,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertFalse(pageData.hasNext()); Assert.assertTrue(pageData.getData().isEmpty()); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); } @Test @@ -518,8 +516,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertFalse(pageData.hasNext()); Assert.assertTrue(pageData.getData().isEmpty()); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); } } diff --git a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java index 5cae75ef0d..63d7243009 100644 --- a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java @@ -93,7 +93,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest @After public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test From 18a7884cbd9a765e64bb438284bc8906b1f4f2bb Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 24 Apr 2024 14:59:02 +0300 Subject: [PATCH 49/72] Improve stability of EdgeEventControllerTest --- .../controller/EdgeEventControllerTest.java | 75 ++++++++++++------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java index cc5e5d5057..e8783ce1b9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java @@ -28,7 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; @@ -89,21 +90,22 @@ public class EdgeEventControllerTest extends AbstractControllerTest { Edge edge = constructEdge("TestEdge", "default"); edge = doPost("/api/edge", edge, Edge.class); + final EdgeId edgeId = edge.getId(); + + awaitForRuleChainToInit(edgeId); + // simulate edge activation ObjectNode attributes = JacksonUtil.newObjectNode(); attributes.put("active", true); - doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + DataConstants.SERVER_SCOPE, attributes); + doPost("/api/plugins/telemetry/EDGE/" + edge.getId() + "/attributes/" + AttributeScope.SERVER_SCOPE, attributes); Device device = constructDevice("TestDevice", "default"); Device savedDevice = doPost("/api/device", device, Device.class); - - final EdgeId edgeId = edge.getId(); - doPost("/api/edge/" + edgeId.toString() + "/device/" + savedDevice.getId().toString(), Device.class); + doPost("/api/edge/" + edgeId + "/device/" + savedDevice.getId(), Device.class); Asset asset = constructAsset("TestAsset", "default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); - - doPost("/api/edge/" + edgeId.toString() + "/asset/" + savedAsset.getId().toString(), Asset.class); + doPost("/api/edge/" + edgeId + "/asset/" + savedAsset.getId(), Asset.class); EntityRelation relation = new EntityRelation(savedAsset.getId(), savedDevice.getId(), EntityRelation.CONTAINS_TYPE); @@ -114,31 +116,12 @@ public class EdgeEventControllerTest extends AbstractControllerTest { awaitForNumberOfEdgeEvents(edgeId, 3); List edgeEvents = findEdgeEvents(edgeId); - Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.DEVICE)); // TestDevice - Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.ASSET)); // TestAsset + Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.DEVICE)); + Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.ASSET)); Assert.assertTrue(popEdgeEvent(edgeEvents, EdgeEventType.RELATION)); Assert.assertTrue(edgeEvents.isEmpty()); } - private boolean popEdgeEvent(List edgeEvents, EdgeEventType edgeEventType) { - for (EdgeEvent edgeEvent : edgeEvents) { - if (edgeEventType.equals(edgeEvent.getType())) { - edgeEvents.remove(edgeEvent); - return true; - } - } - return false; - } - - private void awaitForNumberOfEdgeEvents(EdgeId edgeId, int expectedNumber) { - Awaitility.await() - .atMost(30, TimeUnit.SECONDS) - .until(() -> { - List edgeEvents = findEdgeEvents(edgeId); - return edgeEvents.size() == expectedNumber; - }); - } - @Test public void saveEdgeEvent_thenCreatePartitionIfNotExist() { reset(partitioningRepository); @@ -167,12 +150,46 @@ public class EdgeEventControllerTest extends AbstractControllerTest { }); } + private boolean popEdgeEvent(List edgeEvents, EdgeEventType edgeEventType) { + for (EdgeEvent edgeEvent : edgeEvents) { + if (edgeEventType.equals(edgeEvent.getType())) { + edgeEvents.remove(edgeEvent); + return true; + } + } + return false; + } + + private void awaitForNumberOfEdgeEvents(EdgeId edgeId, int expectedNumber) { + Awaitility.await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> { + List edgeEvents = findEdgeEvents(edgeId); + return edgeEvents.size() == expectedNumber; + }); + } + private List findEdgeEvents(EdgeId edgeId) throws Exception { - return doGetTypedWithTimePageLink("/api/edge/" + edgeId.toString() + "/events?", + return doGetTypedWithTimePageLink("/api/edge/" + edgeId + "/events?", new TypeReference>() { }, new TimePageLink(10)).getData(); } + private void awaitForRuleChainToInit(EdgeId edgeId) { + Awaitility.await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> { + List ruleChains = getEdgeRuleChains(edgeId); + return ruleChains.size() == 1; + }); + } + + private List getEdgeRuleChains(EdgeId edgeId) throws Exception { + return doGetTypedWithTimePageLink("/api/edge/" + edgeId + "/ruleChains?", + new TypeReference>() { + }, new TimePageLink(10)).getData(); + } + private Device constructDevice(String name, String type) { Device device = new Device(); device.setName(name); From 80416a7e33b4d95e2230e4c18110d81f4c257ca3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 24 Apr 2024 17:13:44 +0300 Subject: [PATCH 50/72] Refactor tenant creation in tests --- .../server/controller/AbstractWebTest.java | 10 +++--- .../controller/AssetControllerTest.java | 4 +-- .../AssetProfileControllerTest.java | 2 +- .../controller/AuditLogControllerTest.java | 3 +- .../ComponentDescriptorControllerTest.java | 2 +- .../controller/CustomerControllerTest.java | 16 ++++----- .../controller/DashboardControllerTest.java | 14 ++++---- .../DeviceConnectivityControllerTest.java | 2 +- .../controller/DeviceControllerTest.java | 8 ++--- .../DeviceProfileControllerTest.java | 2 +- .../server/controller/EdgeControllerTest.java | 2 +- .../controller/EntityQueryControllerTest.java | 2 +- .../EntityRelationControllerTest.java | 2 +- .../controller/EntityViewControllerTest.java | 4 +-- .../server/controller/HomePageApiTest.java | 2 +- .../controller/OtaPackageControllerTest.java | 4 +-- .../server/controller/RpcControllerTest.java | 2 +- .../controller/RuleChainControllerTest.java | 8 ++--- .../controller/TbResourceControllerTest.java | 2 +- .../controller/TenantControllerTest.java | 32 ++++++++--------- .../TenantProfileControllerTest.java | 36 +++++++++---------- .../server/controller/UserControllerTest.java | 14 ++++---- .../controller/WidgetTypeControllerTest.java | 21 ++++++----- .../WidgetsBundleControllerTest.java | 2 +- .../server/edge/TenantEdgeTest.java | 4 +-- ...AbstractRuleEngineFlowIntegrationTest.java | 5 ++- .../sql/BaseTbResourceServiceTest.java | 7 ++-- .../SequentialTimeseriesPersistenceTest.java | 4 +-- 28 files changed, 106 insertions(+), 110 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 75d127a29d..732ddb76aa 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -288,7 +288,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { Tenant tenant = new Tenant(); tenant.setTitle(TEST_TENANT_NAME); - Tenant savedTenant = createTenant(tenant); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantId = savedTenant.getId(); tenantProfileId = savedTenant.getTenantProfileId(); @@ -426,7 +426,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle(TEST_DIFFERENT_TENANT_NAME); - savedDifferentTenant = createTenant(tenant); + savedDifferentTenant = saveTenant(tenant); differentTenantId = savedDifferentTenant.getId(); Assert.assertNotNull(savedDifferentTenant); User differentTenantAdmin = new User(); @@ -436,9 +436,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { savedDifferentTenantUser = createUserAndLogin(differentTenantAdmin, DIFFERENT_TENANT_ADMIN_PASSWORD); } - protected Tenant createTenant(Tenant tenant) throws Exception { + protected Tenant saveTenant(Tenant tenant) throws Exception { tenant = doPost("/api/tenant", tenant, Tenant.class); - attributesService.save(TenantId.SYS_TENANT_ID, tenant.getId(), AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker attr to later know when Housekeeper finishes tenant cleanup + if (tenant.getId() == null) { + attributesService.save(TenantId.SYS_TENANT_ID, tenant.getId(), AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker attr to later know when Housekeeper finishes tenant cleanup + } return tenant; } diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java index 9e0ae0ab4c..123696ce3d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java @@ -90,7 +90,7 @@ public class AssetControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -553,7 +553,7 @@ public class AssetControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java index e563b13179..c408daaab3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java @@ -82,7 +82,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index 4059c3a70a..3150edeb37 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class AuditLogControllerTest extends AbstractControllerTest { @@ -75,7 +74,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java index 4b3d1265ea..bfaecc803e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java @@ -47,7 +47,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java index 544096d27e..f431f9ac57 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java @@ -85,7 +85,7 @@ public class CustomerControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -153,7 +153,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString(msgError))); customer.setTenantId(savedTenant.getId()); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -164,7 +164,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -175,7 +175,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -186,7 +186,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -197,7 +197,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -208,7 +208,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @@ -292,7 +292,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java index db7acafdb9..7e1b9b30c9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java @@ -77,7 +77,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -165,7 +165,7 @@ public class DashboardControllerTest extends AbstractControllerTest { @Test public void testSaveDashboardWithEmptyTitle() throws Exception { Dashboard dashboard = new Dashboard(); - String msgError = "Dashboard title " + msgErrorShouldBeSpecified;; + String msgError = "Dashboard title " + msgErrorShouldBeSpecified; Mockito.reset(tbClusterService, auditLogService); @@ -195,7 +195,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Assert.assertTrue(assignedDashboard.getAssignedCustomers().contains(savedCustomer.toShortCustomerInfo())); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(), - savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, + savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, assignedDashboard.getId().getId().toString(), savedCustomer.getId().getId().toString(), savedCustomer.getTitle()); Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); @@ -238,8 +238,8 @@ public class DashboardControllerTest extends AbstractControllerTest { Assert.assertTrue(publicCustomer.isPublic()); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(), - savedTenant.getId(), publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, - ActionType.UPDATED, assignedDashboard .getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle()); + savedTenant.getId(), publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, + ActionType.UPDATED, assignedDashboard.getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle()); Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); Assert.assertTrue(foundDashboard.getAssignedCustomers().contains(publicCustomer.toShortCustomerInfo())); @@ -282,7 +282,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -480,7 +480,7 @@ public class DashboardControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new Dashboard(), new Dashboard(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.ADDED, cntEntity, cntEntity, cntEntity*2); + ActionType.ADDED, cntEntity, cntEntity, cntEntity * 2); List loadedDashboards = new ArrayList<>(); PageLink pageLink = new PageLink(21); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 1bc51e90e2..38e537340f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -142,7 +142,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 1cc1468c68..7e62d9d326 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -132,7 +132,7 @@ public class DeviceControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -686,7 +686,7 @@ public class DeviceControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -911,7 +911,7 @@ public class DeviceControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new Device(), new Device(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity,1); + ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity, 1); testNotificationUpdateGatewayNever(); } @@ -1316,7 +1316,7 @@ public class DeviceControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("Different tenant"); - Tenant savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedDifferentTenant = saveTenant(tenant); Assert.assertNotNull(savedDifferentTenant); User user = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java index a9d15ac3fc..b549d2af2a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java @@ -93,7 +93,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index f8ac2b7dda..49dfa71223 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -361,7 +361,7 @@ public class EdgeControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index f12dad6e4e..e67707b0c8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -83,7 +83,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index 7068eafb61..48e0fb6196 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -72,7 +72,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 2adf5b37b5..e525f0efd5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -370,7 +370,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant2 = getNewTenant("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -430,7 +430,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { testNotifyEntityBroadcastEntityStateChangeEventMany(new EntityView(), new EntityView(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, cntEntity, cntEntity, - cntEntity*2, 3); + cntEntity * 2, 3); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java index 21d6f50748..31c82408c1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java @@ -103,7 +103,7 @@ public class HomePageApiTest extends AbstractControllerTest { for (int i = 0; i < 100; i++) { Tenant tenant = new Tenant(); tenant.setTitle("tenant" + i); - tenants.add(doPost("/api/tenant", tenant, Tenant.class)); + tenants.add(saveTenant(tenant)); } EntityTypeFilter ef = new EntityTypeFilter(); diff --git a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java index 33da6f4a80..4031045095 100644 --- a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java @@ -72,7 +72,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -338,7 +338,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new OtaPackageInfo(), new OtaPackageInfo(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.ADDED, cntEntity, 0, (cntEntity*2 - startIndexSaveData)); + ActionType.ADDED, cntEntity, 0, (cntEntity * 2 - startIndexSaveData)); List loadedFirmwares = new ArrayList<>(); PageLink pageLink = new PageLink(24); diff --git a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java index 8bc32c2103..d0f6426ab1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java @@ -49,7 +49,7 @@ public class RpcControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java index 28b6c75a12..a990f55224 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java @@ -82,7 +82,7 @@ public class RuleChainControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -146,9 +146,9 @@ public class RuleChainControllerTest extends AbstractControllerTest { int currentVersion = annotation.version(); String oldConfig = "{\"attrMapping\":{\"serialNumber\":\"sn\"}," + - "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," + - "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," + - "\"fetchLastLevelOnly\":false},\"telemetry\":false}"; + "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," + + "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," + + "\"fetchLastLevelOnly\":false},\"telemetry\":false}"; TbGetRelatedDataNodeConfiguration defaultConfiguration = new TbGetRelatedDataNodeConfiguration().defaultConfiguration(); String newConfig = JacksonUtil.toString(defaultConfiguration); diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index fad526bb78..0745761c25 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -73,7 +73,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 6edcd77694..19ffb1cd82 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -140,7 +140,7 @@ public class TenantControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); Assert.assertNotNull(savedTenant.getId()); Assert.assertTrue(savedTenant.getCreatedTime() > 0); @@ -149,7 +149,7 @@ public class TenantControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.CREATED, 1); savedTenant.setTitle("My new tenant"); - doPost("/api/tenant", savedTenant, Tenant.class); + saveTenant(savedTenant); Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertEquals(foundTenant.getTitle(), savedTenant.getTitle()); @@ -180,7 +180,7 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(savedTenant, foundTenant); @@ -192,7 +192,7 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); TenantInfo foundTenant = doGet("/api/tenant/info/" + savedTenant.getId().getId().toString(), TenantInfo.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(new TenantInfo(savedTenant, "Default"), foundTenant); @@ -234,7 +234,7 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); String tenantIdStr = savedTenant.getId().getId().toString(); deleteTenant(savedTenant.getId()); @@ -261,7 +261,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + saveTenant(tenant))); } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); @@ -305,7 +305,7 @@ public class TenantControllerTest extends AbstractControllerTest { title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + saveTenant(tenant))); } List tenantsTitle1 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); @@ -320,7 +320,7 @@ public class TenantControllerTest extends AbstractControllerTest { title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + saveTenant(tenant))); } List tenantsTitle2 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); @@ -392,7 +392,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); createFutures.add(executor.submit(() -> - new TenantInfo(doPost("/api/tenant", tenant, Tenant.class), "Default"))); + new TenantInfo(saveTenant(tenant), "Default"))); } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); @@ -444,7 +444,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Isolated tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); User tenantUser = new User(); tenantUser.setAuthority(Authority.TENANT_ADMIN); @@ -473,7 +473,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); tenant.setTenantProfileId(tenantProfile.getId()); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); @@ -503,7 +503,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class); tenant.setTenantProfileId(tenantProfile2.getId()); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); @@ -545,7 +545,7 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); tenant.setTenantProfileId(null); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); for (Queue queue : foundTenantQueues) { @@ -572,7 +572,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; loginDifferentTenant(); @@ -657,7 +657,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; await().atMost(30, TimeUnit.SECONDS) .until(() -> { @@ -712,7 +712,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(partitionService.getMyPartitions(new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId))).isNotNull(); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index 15384a1ed0..989e80e6e5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -75,7 +75,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { savedTenantProfile.setName("New tenant profile"); doPost("/api/tenantProfile", savedTenantProfile, TenantProfile.class); - TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class); + TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class); Assert.assertEquals(foundTenantProfile.getName(), savedTenantProfile.getName()); testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.UPDATED, 1); @@ -100,7 +100,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class); + TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class); Assert.assertNotNull(foundTenantProfile); Assert.assertEquals(savedTenantProfile, foundTenantProfile); } @@ -110,7 +110,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/"+savedTenantProfile.getId().getId().toString(), EntityInfo.class); + EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/" + savedTenantProfile.getId().getId().toString(), EntityInfo.class); Assert.assertNotNull(foundTenantProfileInfo); Assert.assertEquals(savedTenantProfile.getId(), foundTenantProfileInfo.getId()); Assert.assertEquals(savedTenantProfile.getName(), foundTenantProfileInfo.getName()); @@ -129,7 +129,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile 1"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString()+"/default", TenantProfile.class); + TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString() + "/default", TenantProfile.class); Assert.assertNotNull(defaultTenantProfile); EntityInfo foundDefaultTenantProfile = doGet("/api/tenantProfileInfo/default", EntityInfo.class); Assert.assertNotNull(foundDefaultTenantProfile); @@ -176,7 +176,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant with tenant profile"); tenant.setTenantProfileId(savedTenantProfile.getId()); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Mockito.reset(tbClusterService); @@ -213,7 +213,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); tenantProfiles.addAll(pageData.getData()); @@ -221,8 +221,8 @@ public class TenantProfileControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService); int cntEntity = 28; - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); } @@ -232,7 +232,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); do { pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); loadedTenantProfiles.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -255,7 +255,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); @@ -268,13 +268,13 @@ public class TenantProfileControllerTest extends AbstractControllerTest { List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData tenantProfilePageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); Assert.assertFalse(tenantProfilePageData.hasNext()); Assert.assertEquals(1, tenantProfilePageData.getTotalElements()); tenantProfiles.addAll(tenantProfilePageData.getData()); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); } @@ -283,7 +283,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { PageData pageData; do { pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); loadedTenantProfileInfos.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -307,7 +307,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); } @@ -353,14 +353,14 @@ public class TenantProfileControllerTest extends AbstractControllerTest { ArgumentMatcher matcherTenantProfile = cntTime == 1 ? argument -> argument.equals(tenantProfile) : argument -> argument.getClass().equals(TenantProfile.class); if (ComponentLifecycleEvent.DELETED.equals(event)) { - Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileDelete(Mockito.argThat( matcherTenantProfile), + Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile), Mockito.isNull()); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); } else { - Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), + Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), Mockito.isNull()); TenantProfileId tenantProfileIdId = cntTime == 1 ? tenantProfile.getId() : (TenantProfileId) createEntityId_NULL_UUID(tenantProfile); - testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime); + testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime); } Mockito.reset(tbClusterService); } diff --git a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java index 7ad2e1771b..c1f3a90529 100644 --- a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java @@ -380,7 +380,7 @@ public class UserControllerTest extends AbstractControllerTest { //here created a new tenant despite already created on AbstractWebTest and then delete the tenant properly on the last line Tenant tenant = new Tenant(); tenant.setTitle("My tenant with many admins"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); TenantId tenantId = savedTenant.getId(); @@ -668,7 +668,7 @@ public class UserControllerTest extends AbstractControllerTest { List expectedCustomerUserIds = new ArrayList<>(); expectedCustomerUserIds.add(customerUserId); for (int i = 0; i < 45; i++) { - User customerUser = createCustomerUser( customerId); + User customerUser = createCustomerUser(customerId); customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); User user = doPost("/api/user", customerUser, User.class); expectedCustomerUserIds.add(user.getId()); @@ -749,7 +749,7 @@ public class UserControllerTest extends AbstractControllerTest { String email = "testEmail1"; for (int i = 0; i < 45; i++) { - User customerUser = createCustomerUser( customerId); + User customerUser = createCustomerUser(customerId); customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); doPost("/api/user", customerUser, User.class); } @@ -992,8 +992,8 @@ public class UserControllerTest extends AbstractControllerTest { List usersInfo = getUsersInfo(pageLink); List expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), - customerUser.getLastName() == null ? "" : customerUser.getLastName())) + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); usersInfo.sort(userDataIdComparator); @@ -1045,8 +1045,8 @@ public class UserControllerTest extends AbstractControllerTest { List usersInfo = getUsersInfo(pageLink); List expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), - customerUser.getLastName() == null ? "" : customerUser.getLastName())) + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); usersInfo.sort(userDataIdComparator); diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index 19e05fbd79..5c4e2afde5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -24,7 +24,6 @@ import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; @@ -53,7 +52,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -126,10 +125,10 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); - doDelete("/api/widgetType/"+savedWidgetType.getId().getId().toString()) + doDelete("/api/widgetType/" + savedWidgetType.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/widgetType/"+savedWidgetType.getId().getId().toString()) + doGet("/api/widgetType/" + savedWidgetType.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -172,7 +171,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class); List widgetTypes = new ArrayList<>(); - for (int i=0;i<89;i++) { + for (int i = 0; i < 89; i++) { WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setName("Widget Type " + i); widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); @@ -183,7 +182,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { doPost("/api/widgetsBundle/" + widgetsBundle.getId().getId().toString() + "/widgetTypes", widgetTypeIds); List loadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(widgetTypes, idComparator); Collections.sort(loadedWidgetTypes, idComparator); @@ -193,12 +192,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginCustomerUser(); List loadedWidgetTypesCustomer = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(loadedWidgetTypesCustomer, idComparator); Assert.assertEquals(widgetTypes, loadedWidgetTypesCustomer); List customerLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); List widgetTypesFromDetailsListCustomer = customerLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); Collections.sort(widgetTypesFromDetailsListCustomer, idComparator); Assert.assertEquals(widgetTypesFromDetailsListCustomer, loadedWidgetTypes); @@ -206,12 +205,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginSysAdmin(); List sysAdminLoadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(sysAdminLoadedWidgetTypes, idComparator); Assert.assertEquals(widgetTypes, sysAdminLoadedWidgetTypes); List sysAdminLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); List widgetTypesFromDetailsListSysAdmin = sysAdminLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); Collections.sort(widgetTypesFromDetailsListSysAdmin, idComparator); Assert.assertEquals(widgetTypesFromDetailsListSysAdmin, loadedWidgetTypes); @@ -224,7 +223,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); WidgetType foundWidgetType = doGet("/api/widgetType?fqn={fqn}", - WidgetType.class, "tenant."+savedWidgetType.getFqn()); + WidgetType.class, "tenant." + savedWidgetType.getFqn()); Assert.assertNotNull(foundWidgetType); Assert.assertEquals(new WidgetType(savedWidgetType), foundWidgetType); } diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java index 6ff9bf2c6e..03506af95c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java @@ -53,7 +53,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java index 7d740ca472..2611aa433c 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java @@ -40,7 +40,7 @@ public class TenantEdgeTest extends AbstractEdgeTest { // updated edge tenant savedTenant.setTitle("Updated Title for Tenant Edge Test"); edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg - savedTenant = doPost("/api/tenant", savedTenant, Tenant.class); + savedTenant = saveTenant(savedTenant); Assert.assertTrue(edgeImitator.waitForMessages()); Optional tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class); Assert.assertTrue(tenantUpdateMsgOpt.isPresent()); @@ -61,7 +61,7 @@ public class TenantEdgeTest extends AbstractEdgeTest { TenantProfile tenantProfile = createTenantProfile(); savedTenant.setTenantProfileId(tenantProfile.getId()); edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg - savedTenant = doPost("/api/tenant", savedTenant, Tenant.class); + savedTenant = saveTenant(savedTenant); Assert.assertTrue(edgeImitator.waitForMessages()); tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class); Assert.assertTrue(tenantUpdateMsgOpt.isPresent()); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 7238ae8bf6..9e36fed677 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -27,9 +27,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration; -import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; @@ -62,7 +62,6 @@ import java.util.List; import java.util.stream.Collectors; import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** * @author Valerii Sosliuk @@ -103,7 +102,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); ruleChainService.deleteRuleChainsByTenantId(savedTenant.getId()); diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index 34f43512bf..566d272a2f 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -51,7 +51,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class BaseTbResourceServiceTest extends AbstractControllerTest { @@ -129,7 +128,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); tenantId = savedTenant.getId(); Assert.assertNotNull(savedTenant); @@ -396,7 +395,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); TenantId tenantId = tenant.getId(); @@ -449,7 +448,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); TenantId tenantId = tenant.getId(); diff --git a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java index 63d7243009..02281abac3 100644 --- a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java @@ -50,8 +50,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @DaoSqlTest public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest { @@ -77,7 +75,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); From bba7f738eb1f0edfdef67a1fe84a3c4c993c725e Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Wed, 24 Apr 2024 17:23:41 +0300 Subject: [PATCH 51/72] Change method name --- .../thingsboard/server/controller/RuleChainController.java | 5 ----- .../server/controller/EdgeEventControllerTest.java | 4 ++-- .../thingsboard/server/dao/rule/BaseRuleChainService.java | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 680633e103..63a435e6f1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -21,14 +21,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.responses.ApiResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -48,7 +45,6 @@ import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; @@ -56,7 +52,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageDataIterableByTenant; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.DefaultRuleChainCreateRequest; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java index e8783ce1b9..a87ab737e8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeEventControllerTest.java @@ -92,7 +92,7 @@ public class EdgeEventControllerTest extends AbstractControllerTest { final EdgeId edgeId = edge.getId(); - awaitForRuleChainToInit(edgeId); + awaitForEdgeTemplateRootRuleChainToAssignToEdge(edgeId); // simulate edge activation ObjectNode attributes = JacksonUtil.newObjectNode(); @@ -175,7 +175,7 @@ public class EdgeEventControllerTest extends AbstractControllerTest { }, new TimePageLink(10)).getData(); } - private void awaitForRuleChainToInit(EdgeId edgeId) { + private void awaitForEdgeTemplateRootRuleChainToAssignToEdge(EdgeId edgeId) { Awaitility.await() .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 8e1af4a531..cf44b39a60 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -653,7 +653,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC @Override public RuleChain getEdgeTemplateRootRuleChain(TenantId tenantId) { - Validator.validateId(tenantId, "Incorrect tenant id for search request."); + Validator.validateId(tenantId, id -> INCORRECT_TENANT_ID + id); return ruleChainDao.findRootRuleChainByTenantIdAndType(tenantId.getId(), RuleChainType.EDGE); } From 327503fd0a7d509c249a75c11ad05e86d7b7cbc3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 25 Apr 2024 12:03:12 +0300 Subject: [PATCH 52/72] Await all housekeeper tasks to be processed in tests --- .../thingsboard/server/controller/AbstractWebTest.java | 8 ++++++-- .../server/queue/memory/DefaultInMemoryStorage.java | 7 +++++++ .../thingsboard/server/queue/memory/InMemoryStorage.java | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 732ddb76aa..a3a048fe9e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -120,6 +120,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.ClaimDevicesService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest; @@ -246,6 +247,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @SpyBean protected MailService mailService; + @Autowired + protected InMemoryStorage storage; + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -372,8 +376,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } catch (Exception e) { throw new RuntimeException(e); } - Awaitility.await("tenant cleanup finish").atMost(30, TimeUnit.SECONDS) - .until(() -> attributesService.find(TenantId.SYS_TENANT_ID, tenantId, AttributeScope.SERVER_SCOPE, "test").get().isEmpty()); + Awaitility.await("all tasks processed").atMost(TIMEOUT, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) + .until(() -> storage.getLag("tb_housekeeper") == 0); } private List getAllTenants() throws Exception { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java index 9c7dc02b4f..f46a05ca26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java @@ -20,8 +20,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -47,6 +49,11 @@ public final class DefaultInMemoryStorage implements InMemoryStorage { return storage.values().stream().map(BlockingQueue::size).reduce(0, Integer::sum); } + @Override + public int getLag(String topic) { + return Optional.ofNullable(storage.get(topic)).map(Collection::size).orElse(0); + } + @Override public boolean put(String topic, TbQueueMsg msg) { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 0e753d1a81..81b174a0d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -25,6 +25,8 @@ public interface InMemoryStorage { int getLagTotal(); + int getLag(String topic); + boolean put(String topic, TbQueueMsg msg); List get(String topic) throws InterruptedException; From 1756d198200cacddbc8e9ca014a6456e89be0015 Mon Sep 17 00:00:00 2001 From: Oleksandra Matviienko Date: Thu, 25 Apr 2024 13:56:25 +0200 Subject: [PATCH 53/72] JUnit5 migration in rule-engine-components Signed-off-by: Oleksandra Matviienko --- rule-engine/rule-engine-components/pom.xml | 5 - .../action/TbCreateRelationNodeTest.java | 14 +- .../credentials/CertPemCredentialsTest.java | 19 +- .../engine/edge/TbMsgPushToEdgeNodeTest.java | 12 +- .../engine/filter/TbJsFilterNodeTest.java | 10 +- .../engine/filter/TbJsSwitchNodeTest.java | 8 +- .../rule/engine/geo/GeoUtilTest.java | 210 +++++++++--------- .../engine/math/TbMathArgumentValueTest.java | 28 +-- .../metadata/TbGetAttributesNodeTest.java | 40 ++-- .../metadata/TbGetDeviceAttrNodeTest.java | 2 +- .../metadata/TbGetTelemetryNodeTest.java | 6 +- .../rule/engine/profile/AlarmStateTest.java | 8 +- .../rule/engine/profile/DeviceStateTest.java | 6 +- .../profile/TbDeviceProfileNodeTest.java | 14 +- .../engine/rest/TbRestApiCallNodeTest.java | 46 ++-- .../engine/rpc/TbSendRPCReplyNodeTest.java | 12 +- .../transform/TbChangeOriginatorNodeTest.java | 15 +- .../transform/TbTransformMsgNodeTest.java | 10 +- .../rule/engine/util/TenantIdLoaderTest.java | 22 +- .../test/resources/junit-platform.properties | 3 + 20 files changed, 244 insertions(+), 246 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/resources/junit-platform.properties diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 224aea3137..de48596d70 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -138,11 +138,6 @@ spring-boot-starter-test test - - org.junit.vintage - junit-vintage-engine - test - org.awaitility awaitility diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java index ec2b4a581f..5d7b92615d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java @@ -17,12 +17,12 @@ package org.thingsboard.rule.engine.action; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.TestDbCallbackExecutor; @@ -49,13 +49,13 @@ import org.thingsboard.server.dao.relation.RelationService; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbCreateRelationNodeTest { private TbCreateRelationNode node; @@ -74,7 +74,7 @@ public class TbCreateRelationNodeTest { private ListeningExecutor dbExecutor; - @Before + @BeforeEach public void before() { dbExecutor = new TestDbCallbackExecutor(); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 6aa1b02fa9..cd191c3707 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -16,8 +16,7 @@ package org.thingsboard.rule.engine.credentials; import org.apache.commons.io.FileUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -49,14 +48,14 @@ public class CertPemCredentialsTest { List x509Certificates = SslUtil.readCertFile(fileContent); - Assert.assertEquals(4, x509Certificates.size()); - Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + Assertions.assertEquals(4, x509Certificates.size()); + Assertions.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", x509Certificates.get(0).getSubjectDN().getName()); - Assert.assertEquals("CN=Sectigo ECC Organization Validation Secure Server CA, O=Sectigo Limited, L=Salford, ST=Greater Manchester, C=GB", + Assertions.assertEquals("CN=Sectigo ECC Organization Validation Secure Server CA, O=Sectigo Limited, L=Salford, ST=Greater Manchester, C=GB", x509Certificates.get(1).getSubjectDN().getName()); - Assert.assertEquals("CN=USERTrust ECC Certification Authority, O=The USERTRUST Network, L=Jersey City, ST=New Jersey, C=US", + Assertions.assertEquals("CN=USERTrust ECC Certification Authority, O=The USERTRUST Network, L=Jersey City, ST=New Jersey, C=US", x509Certificates.get(2).getSubjectDN().getName()); - Assert.assertEquals("CN=AAA Certificate Services, O=Comodo CA Limited, L=Salford, ST=Greater Manchester, C=GB", + Assertions.assertEquals("CN=AAA Certificate Services, O=Comodo CA Limited, L=Salford, ST=Greater Manchester, C=GB", x509Certificates.get(3).getSubjectDN().getName()); } @@ -66,8 +65,8 @@ public class CertPemCredentialsTest { List x509Certificates = SslUtil.readCertFile(fileContent); - Assert.assertEquals(1, x509Certificates.size()); - Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + Assertions.assertEquals(1, x509Certificates.size()); + Assertions.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", x509Certificates.get(0).getSubjectDN().getName()); } @@ -77,7 +76,7 @@ public class CertPemCredentialsTest { List x509Certificates = SslUtil.readCertFile(fileContent); - Assert.assertEquals(0, x509Certificates.size()); + Assertions.assertEquals(0, x509Certificates.size()); } private static Stream testLoadKeyStore() { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNodeTest.java index 5f57f7f5e3..b8a70991bc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/edge/TbMsgPushToEdgeNodeTest.java @@ -16,13 +16,13 @@ package org.thingsboard.rule.engine.edge; import com.google.common.util.concurrent.SettableFuture; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentMatcher; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.TbContext; @@ -50,7 +50,7 @@ import java.util.UUID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbMsgPushToEdgeNodeTest { private static final List MISC_EVENTS = List.of(TbMsgType.CONNECT_EVENT, TbMsgType.DISCONNECT_EVENT, @@ -71,7 +71,7 @@ public class TbMsgPushToEdgeNodeTest { @Mock private ListeningExecutor dbCallbackExecutor; - @Before + @BeforeEach public void setUp() throws TbNodeException { node = new TbMsgPushToEdgeNode(); TbMsgPushToEdgeNodeConfiguration config = new TbMsgPushToEdgeNodeConfiguration().defaultConfiguration(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java index 903944025a..b991b7b220 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsFilterNodeTest.java @@ -17,11 +17,11 @@ package org.thingsboard.rule.engine.filter; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -38,12 +38,12 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import javax.script.ScriptException; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbJsFilterNodeTest { private TbJsFilterNode node; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java index d679e42ba0..ff7a40644b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/filter/TbJsSwitchNodeTest.java @@ -18,10 +18,10 @@ package org.thingsboard.rule.engine.filter; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -38,7 +38,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbJsSwitchNodeTest { private TbJsSwitchNode node; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/geo/GeoUtilTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/geo/GeoUtilTest.java index 2172288236..8d39105d30 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/geo/GeoUtilTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/geo/GeoUtilTest.java @@ -15,12 +15,12 @@ */ package org.thingsboard.rule.engine.geo; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.junit.MockitoJUnitRunner; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class GeoUtilTest { public static final String SIMPLE_RECT = "[[51.903762928405555,23.642220786948297],[44.669801219635644,41.83345155830211]]"; @@ -47,117 +47,117 @@ public class GeoUtilTest { @Test public void testPointsInSimplePolygons() { - Assert.assertTrue("Polygon " + SIMPLE_RECT + " must contain the dot " + POINT_INSIDE_SIMPLE_RECT_CENTER, - GeoUtil.contains(SIMPLE_RECT, POINT_INSIDE_SIMPLE_RECT_CENTER) - ); - Assert.assertTrue("Polygon " + SIMPLE_RECT + " must contain the dot " + POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER, - GeoUtil.contains(SIMPLE_RECT, POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER) - ); - Assert.assertTrue("Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must contain the dot " - + POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER, - GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER) - ); + Assertions.assertTrue(GeoUtil.contains(SIMPLE_RECT, POINT_INSIDE_SIMPLE_RECT_CENTER), + "Polygon " + SIMPLE_RECT + " must contain the dot " + POINT_INSIDE_SIMPLE_RECT_CENTER + ); + Assertions.assertTrue(GeoUtil.contains(SIMPLE_RECT, POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER), + "Polygon " + SIMPLE_RECT + " must contain the dot " + POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER + ); + Assertions.assertTrue(GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER), + "Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must contain the dot " + + POINT_INSIDE_SIMPLE_RECT_NEAR_BORDER + ); - Assert.assertFalse("Polygon " + SIMPLE_RECT + " must not contain the dot " - + POINT_OUTSIDE_SIMPLE_RECT, - GeoUtil.contains(SIMPLE_RECT, POINT_OUTSIDE_SIMPLE_RECT) - ); - Assert.assertFalse("Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must not contain the dot " - + POINT_OUTSIDE_SIMPLE_RECT, - GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SIMPLE_RECT) - ); - Assert.assertFalse("Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must not contain the dot " - + POINT_INSIDE_SIMPLE_RECT_CENTER, - GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_INSIDE_SIMPLE_RECT_CENTER) - ); + Assertions.assertFalse(GeoUtil.contains(SIMPLE_RECT, POINT_OUTSIDE_SIMPLE_RECT), + "Polygon " + SIMPLE_RECT + " must not contain the dot " + + POINT_OUTSIDE_SIMPLE_RECT + ); + Assertions.assertFalse(GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SIMPLE_RECT), + "Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must not contain the dot " + + POINT_OUTSIDE_SIMPLE_RECT + ); + Assertions.assertFalse(GeoUtil.contains(SIMPLE_RECT_WITH_HOLE_IN_CENTER, POINT_INSIDE_SIMPLE_RECT_CENTER), + "Polygon " + SIMPLE_RECT_WITH_HOLE_IN_CENTER + " must not contain the dot " + + POINT_INSIDE_SIMPLE_RECT_CENTER + ); } @Test public void testPointsInComplexPolygons() { - Assert.assertTrue("Polygon " + SAND_CLOCK + " must contain the dot " + POINT_INSIDE_SAND_CLOCK_CENTER, - GeoUtil.contains(SAND_CLOCK, POINT_INSIDE_SAND_CLOCK_CENTER) - ); - Assert.assertTrue("Polygon " + SAND_CLOCK + " must contain the dot " + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER, - GeoUtil.contains(SAND_CLOCK, POINT_INSIDE_SAND_CLOCK_NEAR_BORDER) - ); - Assert.assertTrue("Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must contain the dot " - + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER, - GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_INSIDE_SAND_CLOCK_NEAR_BORDER) - ); + Assertions.assertTrue(GeoUtil.contains(SAND_CLOCK, POINT_INSIDE_SAND_CLOCK_CENTER), + "Polygon " + SAND_CLOCK + " must contain the dot " + POINT_INSIDE_SAND_CLOCK_CENTER + ); + Assertions.assertTrue(GeoUtil.contains(SAND_CLOCK, POINT_INSIDE_SAND_CLOCK_NEAR_BORDER), + "Polygon " + SAND_CLOCK + " must contain the dot " + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER + ); + Assertions.assertTrue(GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_INSIDE_SAND_CLOCK_NEAR_BORDER), + "Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must contain the dot " + + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER + ); - Assert.assertFalse("Polygon " + SAND_CLOCK + " must not contain the dot " - + POINT_OUTSIDE_SAND_CLOCK_1, - GeoUtil.contains(SAND_CLOCK, POINT_OUTSIDE_SAND_CLOCK_1) - ); - Assert.assertFalse("Polygon " + SAND_CLOCK + " must not contain the dot " - + POINT_OUTSIDE_SAND_CLOCK_2, - GeoUtil.contains(SAND_CLOCK, POINT_OUTSIDE_SAND_CLOCK_2) - ); - Assert.assertFalse("Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " - + POINT_INSIDE_SAND_CLOCK_CENTER, - GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_INSIDE_SAND_CLOCK_CENTER) - ); - Assert.assertFalse("Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " - + POINT_OUTSIDE_SAND_CLOCK_1, - GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SAND_CLOCK_1) - ); - Assert.assertFalse("Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " - + POINT_OUTSIDE_SAND_CLOCK_2, - GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SAND_CLOCK_2) - ); + Assertions.assertFalse(GeoUtil.contains(SAND_CLOCK, POINT_OUTSIDE_SAND_CLOCK_1), + "Polygon " + SAND_CLOCK + " must not contain the dot " + + POINT_OUTSIDE_SAND_CLOCK_1 + ); + Assertions.assertFalse(GeoUtil.contains(SAND_CLOCK, POINT_OUTSIDE_SAND_CLOCK_2), + "Polygon " + SAND_CLOCK + " must not contain the dot " + + POINT_OUTSIDE_SAND_CLOCK_2 + ); + Assertions.assertFalse(GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_INSIDE_SAND_CLOCK_CENTER), + "Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " + + POINT_INSIDE_SAND_CLOCK_CENTER + ); + Assertions.assertFalse(GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SAND_CLOCK_1), + "Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " + + POINT_OUTSIDE_SAND_CLOCK_1 + ); + Assertions.assertFalse(GeoUtil.contains(SAND_CLOCK_WITH_HOLE_IN_CENTER, POINT_OUTSIDE_SAND_CLOCK_2), + "Polygon " + SAND_CLOCK_WITH_HOLE_IN_CENTER + " must not contain the dot " + + POINT_OUTSIDE_SAND_CLOCK_2 + ); } @Test public void testPointsInSelfIntersectingPolygons() { - Assert.assertTrue("Polygon " + SELF_INTERSECTING + " must contain the dot " - + POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER, - GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER) - ); - Assert.assertTrue("Polygon " + SELF_INTERSECTING + " must contain the dot " - + POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER, - GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER) - ); - Assert.assertTrue("Polygon " + SELF_INTERSECTING + " must contain the dot " - + POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER, - GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER) - ); - Assert.assertTrue("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must contain the dot " - + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER) - ); + Assertions.assertTrue(GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER), + "Polygon " + SELF_INTERSECTING + " must contain the dot " + + POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER + ); + Assertions.assertTrue(GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER), + "Polygon " + SELF_INTERSECTING + " must contain the dot " + + POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER + ); + Assertions.assertTrue(GeoUtil.contains(SELF_INTERSECTING, POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER), + "Polygon " + SELF_INTERSECTING + " must contain the dot " + + POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER + ); + Assertions.assertTrue(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_NEAR_BORDER), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must contain the dot " + + POINT_INSIDE_SAND_CLOCK_NEAR_BORDER + ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_1, - GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_1) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_2, - GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_2) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_3, - GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_3) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_1, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_1) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_2, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_2) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " - + POINT_OUTSIDE_SELF_INTERSECTING_3, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_3) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " - + POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER) - ); - Assert.assertFalse("Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " - + POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER, - GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER) - ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_1), + "Polygon " + SELF_INTERSECTING + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_1 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_2), + "Polygon " + SELF_INTERSECTING + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_2 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING, POINT_OUTSIDE_SELF_INTERSECTING_3), + "Polygon " + SELF_INTERSECTING + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_3 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_1), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_1 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_2), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_2 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_OUTSIDE_SELF_INTERSECTING_3), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " + + POINT_OUTSIDE_SELF_INTERSECTING_3 + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " + + POINT_INSIDE_SELF_INTERSECTING_UPPER_CENTER + ); + Assertions.assertFalse(GeoUtil.contains(SELF_INTERSECTING_WITH_HOLES, POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER), + "Polygon " + SELF_INTERSECTING_WITH_HOLES + " must not contain the dot " + + POINT_INSIDE_SELF_INTERSECTING_LOWER_CENTER + ); } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathArgumentValueTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathArgumentValueTest.java index 5a6fffbb18..073a9fb393 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathArgumentValueTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathArgumentValueTest.java @@ -16,8 +16,8 @@ package org.thingsboard.rule.engine.math; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -32,7 +32,7 @@ public class TbMathArgumentValueTest { TbMathArgument tbMathArgument = new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "TestKey"); tbMathArgument.setDefaultValue(5.0); TbMathArgumentValue result = TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.ofNullable(JacksonUtil.newObjectNode())); - Assert.assertEquals(5.0, result.getValue(), 0d); + Assertions.assertEquals(5.0, result.getValue(), 0d); } @Test @@ -41,14 +41,14 @@ public class TbMathArgumentValueTest { Throwable thrown = assertThrows(RuntimeException.class, () -> { TbMathArgumentValue result = TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.empty()); }); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test public void test_fromMessageBody_then_noKey() { TbMathArgument tbMathArgument = new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "TestKey"); Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.ofNullable(JacksonUtil.newObjectNode()))); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test @@ -59,12 +59,12 @@ public class TbMathArgumentValueTest { //null value Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.of(msgData))); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); //empty value msgData.put("TestKey", ""); thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.of(msgData))); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test @@ -75,40 +75,40 @@ public class TbMathArgumentValueTest { //string value Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.of(msgData))); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); //object value msgData.set("TestKey", JacksonUtil.newObjectNode()); thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageBody(tbMathArgument, tbMathArgument.getKey(), Optional.of(msgData))); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test public void test_fromMessageMetadata_then_noKey() { TbMathArgument tbMathArgument = new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "TestKey"); Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageMetadata(tbMathArgument, tbMathArgument.getKey(), new TbMsgMetaData())); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test public void test_fromMessageMetadata_then_valueEmpty() { TbMathArgument tbMathArgument = new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "TestKey"); Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromMessageMetadata(tbMathArgument, tbMathArgument.getKey(), null)); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } @Test public void test_fromString_thenOK() { var value = "5.0"; TbMathArgumentValue result = TbMathArgumentValue.fromString(value); - Assert.assertNotNull(result); - Assert.assertEquals(5.0, result.getValue(), 0d); + Assertions.assertNotNull(result); + Assertions.assertEquals(5.0, result.getValue(), 0d); } @Test public void test_fromString_then_failure() { var value = "Test"; Throwable thrown = assertThrows(RuntimeException.class, () -> TbMathArgumentValue.fromString(value)); - Assert.assertNotNull(thrown.getMessage()); + Assertions.assertNotNull(thrown.getMessage()); } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java index 26b9c9a30d..9eb74b5627 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetAttributesNodeTest.java @@ -18,14 +18,14 @@ package org.thingsboard.rule.engine.metadata; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.Futures; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; @@ -33,7 +33,6 @@ import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -55,17 +54,18 @@ import java.util.List; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbGetAttributesNodeTest { private static final EntityId ORIGINATOR = new DeviceId(Uuids.timeBased()); @@ -86,7 +86,7 @@ public class TbGetAttributesNodeTest { private long ts; private TbGetAttributesNode node; - @Before + @BeforeEach public void before() throws TbNodeException { dbExecutor = new AbstractListeningExecutor() { @Override @@ -96,10 +96,10 @@ public class TbGetAttributesNodeTest { }; dbExecutor.init(); - when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); - when(ctxMock.getTimeseriesService()).thenReturn(timeseriesServiceMock); - when(ctxMock.getTenantId()).thenReturn(TENANT_ID); - when(ctxMock.getDbCallbackExecutor()).thenReturn(dbExecutor); + lenient().when(ctxMock.getAttributesService()).thenReturn(attributesServiceMock); + lenient().when(ctxMock.getTimeseriesService()).thenReturn(timeseriesServiceMock); + lenient().when(ctxMock.getTenantId()).thenReturn(TENANT_ID); + lenient().when(ctxMock.getDbCallbackExecutor()).thenReturn(dbExecutor); clientAttributes = getAttributeNames("client"); serverAttributes = getAttributeNames("server"); @@ -107,20 +107,20 @@ public class TbGetAttributesNodeTest { tsKeys = List.of("temperature", "humidity", "unknown"); ts = System.currentTimeMillis(); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.CLIENT_SCOPE, clientAttributes)) + lenient().when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.CLIENT_SCOPE, clientAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(clientAttributes, ts))); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SERVER_SCOPE, serverAttributes)) + lenient().when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SERVER_SCOPE, serverAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(serverAttributes, ts))); - when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SHARED_SCOPE, sharedAttributes)) + lenient().when(attributesServiceMock.find(TENANT_ID, ORIGINATOR, AttributeScope.SHARED_SCOPE, sharedAttributes)) .thenReturn(Futures.immediateFuture(getListAttributeKvEntry(sharedAttributes, ts))); - when(timeseriesServiceMock.findLatest(TENANT_ID, ORIGINATOR, tsKeys)) + lenient().when(timeseriesServiceMock.findLatest(TENANT_ID, ORIGINATOR, tsKeys)) .thenReturn(Futures.immediateFuture(getListTsKvEntry(tsKeys, ts))); } - @After + @AfterEach public void after() { dbExecutor.destroy(); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNodeTest.java index f40ab2bed8..ac6bb38be5 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetDeviceAttrNodeTest.java @@ -16,7 +16,7 @@ package org.thingsboard.rule.engine.metadata; import com.fasterxml.jackson.databind.JsonNode; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.util.TbPair; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java index 990688611e..6ecd1b6eb9 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNodeTest.java @@ -15,8 +15,8 @@ */ package org.thingsboard.rule.engine.metadata; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; @@ -37,7 +37,7 @@ public class TbGetTelemetryNodeTest { TbNodeConfiguration nodeConfiguration; TbContext ctx; - @Before + @BeforeEach public void setUp() throws Exception { ctx = mock(TbContext.class); node = spy(new TbGetTelemetryNode()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/AlarmStateTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/AlarmStateTest.java index 7cc6781abb..08d445b89d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/AlarmStateTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/AlarmStateTest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.rule.engine.profile; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.device.profile.AlarmCondition; import org.thingsboard.server.common.data.device.profile.AlarmConditionSpec; @@ -28,9 +28,9 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; public class AlarmStateTest { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java index 0d2002661f..5cee1aa5b3 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java @@ -16,8 +16,8 @@ package org.thingsboard.rule.engine.profile; import com.google.common.util.concurrent.Futures; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; @@ -69,7 +69,7 @@ public class DeviceStateTest { private TbContext ctx; - @Before + @BeforeEach public void beforeEach() { ctx = mock(TbContext.class); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index 645613f553..19eabf0bdc 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -18,11 +18,11 @@ package org.thingsboard.rule.engine.profile; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineDeviceProfileCache; @@ -86,7 +86,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbDeviceProfileNodeTest { private TbDeviceProfileNode node; @@ -1700,10 +1700,10 @@ public class TbDeviceProfileNodeTest { private void init() throws TbNodeException { Mockito.when(ctx.getTenantId()).thenReturn(tenantId); Mockito.when(ctx.getDeviceProfileCache()).thenReturn(cache); - Mockito.when(ctx.getTimeseriesService()).thenReturn(timeseriesService); - Mockito.when(ctx.getAlarmService()).thenReturn(alarmService); + Mockito.lenient().when(ctx.getTimeseriesService()).thenReturn(timeseriesService); + Mockito.lenient().when(ctx.getAlarmService()).thenReturn(alarmService); Mockito.when(ctx.getDeviceService()).thenReturn(deviceService); - Mockito.when(ctx.getAttributesService()).thenReturn(attributesService); + Mockito.lenient().when(ctx.getAttributesService()).thenReturn(attributesService); TbNodeConfiguration nodeConfiguration = new TbNodeConfiguration(JacksonUtil.newObjectNode()); node = new TbDeviceProfileNode(); node.init(ctx, nodeConfiguration); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeTest.java index 78c68a24be..98695c1570 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeTest.java @@ -25,13 +25,13 @@ import org.apache.http.impl.bootstrap.HttpServer; import org.apache.http.impl.bootstrap.ServerBootstrap; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpRequestHandler; -import org.junit.After; -import org.junit.Test; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Assertions; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; @@ -51,12 +51,12 @@ import java.util.Collections; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotSame; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verify; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbRestApiCallNodeTest { private TbRestApiCallNode restNode; @@ -91,7 +91,7 @@ public class TbRestApiCallNodeTest { } } - @After + @AfterEach public void teardown() { if (server != null) { server.stop(); @@ -108,9 +108,9 @@ public class TbRestApiCallNodeTest { public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { try { - assertEquals("Request path matches", request.getRequestLine().getUri(), path); - assertTrue("Custom header included", request.containsHeader("Foo")); - assertEquals("Custom header value", "Bar", request.getFirstHeader("Foo").getValue()); + assertEquals(request.getRequestLine().getUri(), path, "Request path matches"); + assertTrue(request.containsHeader("Foo"), "Custom header included"); + assertEquals("Bar", request.getFirstHeader("Foo").getValue(), "Custom header value"); response.setStatusCode(200); new Thread(new Runnable() { @Override @@ -142,7 +142,7 @@ public class TbRestApiCallNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, originator, metaData, TbMsgDataType.JSON, TbMsg.EMPTY_JSON_OBJECT, ruleChainId, ruleNodeId); restNode.onMsg(ctx, msg); - assertTrue("Server handled request", latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(10, TimeUnit.SECONDS), "Server handled request"); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(TbMsgMetaData.class); @@ -163,15 +163,15 @@ public class TbRestApiCallNodeTest { public void handle(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException, IOException { try { - assertEquals("Request path matches", path, request.getRequestLine().getUri()); - assertTrue("Content-Type included", request.containsHeader("Content-Type")); - assertEquals("Content-Type value", "text/plain;charset=UTF-8", - request.getFirstHeader("Content-Type").getValue()); - assertTrue("Content-Length included", request.containsHeader("Content-Length")); - assertEquals("Content-Length value", "2", - request.getFirstHeader("Content-Length").getValue()); - assertTrue("Custom header included", request.containsHeader("Foo")); - assertEquals("Custom header value", "Bar", request.getFirstHeader("Foo").getValue()); + assertEquals(path, request.getRequestLine().getUri(), "Request path matches"); + assertTrue(request.containsHeader("Content-Type"), "Content-Type included"); + assertEquals("text/plain;charset=UTF-8", + request.getFirstHeader("Content-Type").getValue(), "Content-Type value"); + assertTrue(request.containsHeader("Content-Length"), "Content-Length included"); + assertEquals("2", + request.getFirstHeader("Content-Length").getValue(), "Content-Length value"); + assertTrue(request.containsHeader("Foo"), "Custom header included"); + assertEquals("Bar", request.getFirstHeader("Foo").getValue(), "Custom header value"); response.setStatusCode(200); new Thread(new Runnable() { @Override @@ -203,7 +203,7 @@ public class TbRestApiCallNodeTest { TbMsg msg = TbMsg.newMsg(TbMsgType.POST_TELEMETRY_REQUEST, originator, metaData, TbMsgDataType.JSON, TbMsg.EMPTY_JSON_OBJECT, ruleChainId, ruleNodeId); restNode.onMsg(ctx, msg); - assertTrue("Server handled request", latch.await(10, TimeUnit.SECONDS)); + assertTrue(latch.await(10, TimeUnit.SECONDS), "Server handled request"); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(TbMsgMetaData.class); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java index 454c1a9fe6..db632fe3c7 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rpc/TbSendRPCReplyNodeTest.java @@ -16,12 +16,12 @@ package org.thingsboard.rule.engine.rpc; import com.google.common.util.concurrent.SettableFuture; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.api.RuleEngineRpcService; @@ -43,7 +43,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbSendRPCReplyNodeTest { private static final String DUMMY_SERVICE_ID = "testServiceId"; @@ -68,7 +68,7 @@ public class TbSendRPCReplyNodeTest { @Mock private ListeningExecutor listeningExecutor; - @Before + @BeforeEach public void setUp() throws TbNodeException { node = new TbSendRPCReplyNode(); TbSendRpcReplyNodeConfiguration config = new TbSendRpcReplyNodeConfiguration().defaultConfiguration(); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index e9803289bb..068858996b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -17,12 +17,12 @@ package org.thingsboard.rule.engine.transform; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ListeningExecutor; import org.thingsboard.rule.engine.TestDbCallbackExecutor; @@ -43,14 +43,15 @@ import org.thingsboard.server.dao.asset.AssetService; import java.util.NoSuchElementException; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +//@ExtendWith(MockitoExtension.class) +@ExtendWith(MockitoExtension.class) public class TbChangeOriginatorNodeTest { private static final String CUSTOMER_SOURCE = "CUSTOMER"; @@ -64,7 +65,7 @@ public class TbChangeOriginatorNodeTest { private ListeningExecutor dbExecutor; - @Before + @BeforeEach public void before() { dbExecutor = new TestDbCallbackExecutor(); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java index a410b84532..98ecc237b6 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbTransformMsgNodeTest.java @@ -17,11 +17,11 @@ package org.thingsboard.rule.engine.transform; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.rule.engine.api.TbContext; @@ -37,12 +37,12 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import java.util.Collections; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbTransformMsgNodeTest { private TbTransformMsgNode node; diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java index e06be15fe6..47fa03e08e 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/util/TenantIdLoaderTest.java @@ -15,13 +15,13 @@ */ package org.thingsboard.rule.engine.util; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; import org.thingsboard.common.util.AbstractListeningExecutor; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; import org.thingsboard.rule.engine.api.RuleEngineApiUsageStateService; @@ -88,7 +88,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TenantIdLoaderTest { @Mock @@ -145,7 +145,7 @@ public class TenantIdLoaderTest { private NotificationId notificationId; private AbstractListeningExecutor dbExecutor; - @Before + @BeforeEach public void before() { dbExecutor = new AbstractListeningExecutor() { @Override @@ -165,7 +165,7 @@ public class TenantIdLoaderTest { } } - @After + @AfterEach public void after() { dbExecutor.destroy(); } @@ -384,9 +384,9 @@ public class TenantIdLoaderTest { TenantId targetTenantId = TenantIdLoader.findTenantId(ctx, entityId); String msg = "Check entity type <" + entityType.name() + ">:"; if (equals) { - Assert.assertEquals(msg, targetTenantId, checkTenantId); + Assertions.assertEquals(targetTenantId, checkTenantId, msg); } else { - Assert.assertNotEquals(msg, targetTenantId, checkTenantId); + Assertions.assertNotEquals(targetTenantId, checkTenantId, msg); } } } diff --git a/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties b/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties new file mode 100644 index 0000000000..ad19ea833b --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties @@ -0,0 +1,3 @@ +junit.jupiter.execution.parallel.enabled = true +junit.jupiter.execution.parallel.mode.default = concurrent +junit.jupiter.execution.parallel.mode.classes.default = concurrent \ No newline at end of file From 732c6d76e4100c73c12a56a5a24393e2cee42fe9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 25 Apr 2024 15:53:36 +0300 Subject: [PATCH 54/72] Deleted test testUnassignAlarmOnCustomerRemoving (customer's devices are deleted when customer is deleted, thus all alarms for the device also should be deleted, instead of unassigned) --- .../controller/AlarmControllerTest.java | 55 ------------------- 1 file changed, 55 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 3d0b0f6ea6..7ae283cd46 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -642,61 +642,6 @@ public class AlarmControllerTest extends AbstractControllerTest { }); } - @Test - public void testUnassignAlarmOnCustomerRemoving() throws Exception { - createDifferentTenantCustomer(); - loginDifferentTenant(); - - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setTenantId(tenantId); - user.setCustomerId(differentTenantCustomerId); - user.setEmail("customerForAssign@thingsboard.org"); - User savedUser = createUser(user, "password"); - - Device device = createDevice("Different customer device", "default", "differentTenantTest"); - - Device assignedDevice = doPost("/api/customer/" + differentTenantCustomerId.getId() - + "/device/" + device.getId().getId(), Device.class); - Assert.assertEquals(differentTenantCustomerId, assignedDevice.getCustomerId()); - - Alarm alarm = Alarm.builder() - .type(TEST_ALARM_TYPE) - .tenantId(savedDifferentTenant.getId()) - .customerId(differentTenantCustomerId) - .originator(device.getId()) - .severity(AlarmSeverity.MAJOR) - .build(); - alarm = doPost("/api/alarm", alarm, Alarm.class); - Assert.assertNotNull(alarm); - AlarmId alarmId = alarm.getId(); - - alarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(alarm); - - Mockito.reset(tbClusterService, auditLogService); - long beforeAssignmentTs = System.currentTimeMillis(); - - doPost("/api/alarm/" + alarmId.getId() + "/assign/" + savedUser.getId().getId()).andExpect(status().isOk()); - AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(foundAlarm); - Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - - long afterAssignmentTs = System.currentTimeMillis(); - - Mockito.reset(tbClusterService, auditLogService); - - doDelete("/api/customer/" + differentTenantCustomerId.getId()).andExpect(status().isOk()); - - Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { - AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(alarmInfo); - Assert.assertNull(alarmInfo.getAssigneeId()); - Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); - }); - } - @Test public void testFindAlarmsViaCustomerUser() throws Exception { loginCustomerUser(); From 3ddda71e1430ce8b26ff5f677f93dd53eb48c713 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 25 Apr 2024 15:54:22 +0300 Subject: [PATCH 55/72] Increase timeout for tasks processing in tests --- .../thingsboard/server/controller/AbstractWebTest.java | 2 +- .../server/controller/TenantControllerTest.java | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index a3a048fe9e..706d3a3e5e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -376,7 +376,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } catch (Exception e) { throw new RuntimeException(e); } - Awaitility.await("all tasks processed").atMost(TIMEOUT, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) + Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) .until(() -> storage.getLag("tb_housekeeper") == 0); } diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 19ffb1cd82..3160d542f1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -260,8 +260,7 @@ public class TenantControllerTest extends AbstractControllerTest { for (int i = 0; i < cntEntity; i++) { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); - createFutures.add(executor.submit(() -> - saveTenant(tenant))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); @@ -304,8 +303,7 @@ public class TenantControllerTest extends AbstractControllerTest { String title = title1 + suffix; title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); - createFutures.add(executor.submit(() -> - saveTenant(tenant))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } List tenantsTitle1 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); @@ -319,8 +317,7 @@ public class TenantControllerTest extends AbstractControllerTest { String title = title2 + suffix; title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); - createFutures.add(executor.submit(() -> - saveTenant(tenant))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } List tenantsTitle2 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); From 87fc91e08869df29fed19f04ca9a97c9fd1784ad Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 25 Apr 2024 16:00:18 +0300 Subject: [PATCH 56/72] deleted "required" from tenant email swagger schema --- .../main/java/org/thingsboard/server/common/data/Tenant.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index 7cb7e15395..b7fade3e09 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -152,7 +152,7 @@ public class Tenant extends ContactBased implements HasTenantId, HasTi return super.getPhone(); } - @Schema(requiredMode = Schema.RequiredMode.REQUIRED, description = "Email", example = "example@company.com") + @Schema(description = "Email", example = "example@company.com") @Override public String getEmail() { return super.getEmail(); From 8c92d7bf6e02ae7889887f2c0980585805c6de1a Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 25 Apr 2024 15:34:35 +0200 Subject: [PATCH 57/72] docker images upgraded --- .../server/dao/AbstractRedisContainer.java | 2 +- docker/docker-compose.hybrid.yml | 2 +- docker/docker-compose.kafka.yml | 2 +- docker/docker-compose.redis-cluster.yml | 12 ++++++------ docker/docker-compose.redis-sentinel.yml | 6 +++--- docker/docker-compose.redis.yml | 2 +- docker/docker-compose.yml | 2 +- .../src/test/resources/docker-compose.redis-ssl.yml | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java index 0c0a1c8ef8..d1f3839cf5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java +++ b/dao/src/test/java/org/thingsboard/server/dao/AbstractRedisContainer.java @@ -26,7 +26,7 @@ import java.util.List; public class AbstractRedisContainer { @ClassRule(order = 0) - public static GenericContainer redis = new GenericContainer("redis:7.0") + public static GenericContainer redis = new GenericContainer("redis:7.2") .withExposedPorts(6379); @ClassRule(order = 1) diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index a68e77ed28..43f1e81cd1 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -29,7 +29,7 @@ services: - ./tb-node/postgres:/var/lib/postgresql/data cassandra: restart: always - image: "cassandra:4.0.4" + image: "cassandra:4.1" ports: - "9042" volumes: diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 99af42965b..c842cc75af 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -19,7 +19,7 @@ version: '3.0' services: kafka: restart: always - image: "bitnami/kafka:3.2.0" + image: "bitnami/kafka:3.7.0" ports: - "9092:9092" env_file: diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 6017f2d6df..7b5d6d6dde 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -19,7 +19,7 @@ version: '3.0' services: # Redis cluster redis-node-0: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: @@ -28,7 +28,7 @@ services: - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-1: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-1:/bitnami/redis/data depends_on: @@ -39,7 +39,7 @@ services: - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-2: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-2:/bitnami/redis/data depends_on: @@ -50,7 +50,7 @@ services: - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-3: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-3:/bitnami/redis/data depends_on: @@ -61,7 +61,7 @@ services: - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-4: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data depends_on: @@ -72,7 +72,7 @@ services: - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: - image: bitnami/redis-cluster:7.0 + image: bitnami/redis-cluster:7.2 volumes: - ./tb-node/redis-cluster-data-5:/bitnami/redis/data depends_on: diff --git a/docker/docker-compose.redis-sentinel.yml b/docker/docker-compose.redis-sentinel.yml index 54a3789664..a131bc7df9 100644 --- a/docker/docker-compose.redis-sentinel.yml +++ b/docker/docker-compose.redis-sentinel.yml @@ -19,7 +19,7 @@ version: '3.0' services: # Redis sentinel redis-master: - image: 'bitnami/redis:7.0' + image: 'bitnami/redis:7.2' volumes: - ./tb-node/redis-sentinel-data-master:/bitnami/redis/data environment: @@ -27,7 +27,7 @@ services: - 'REDIS_PASSWORD=thingsboard' redis-slave: - image: 'bitnami/redis:7.0' + image: 'bitnami/redis:7.2' volumes: - ./tb-node/redis-sentinel-data-slave:/bitnami/redis/data environment: @@ -39,7 +39,7 @@ services: - redis-master redis-sentinel: - image: 'bitnami/redis-sentinel:7.0' + image: 'bitnami/redis-sentinel:7.2' volumes: - ./tb-node/redis-sentinel-data-sentinel:/bitnami/redis/data environment: diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml index 60170d5eb6..16b16d4a8e 100644 --- a/docker/docker-compose.redis.yml +++ b/docker/docker-compose.redis.yml @@ -20,7 +20,7 @@ services: # Redis standalone redis: restart: always - image: bitnami/redis:7.0 + image: bitnami/redis:7.2 environment: # ALLOW_EMPTY_PASSWORD is recommended only for development. ALLOW_EMPTY_PASSWORD: "yes" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 18e80b425d..3205c62e57 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -301,7 +301,7 @@ services: haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" - image: thingsboard/haproxy-certbot:2.2.31-alpine3.18 + image: thingsboard/haproxy-certbot:2.2.33-alpine volumes: - ./haproxy/config:/config - ./haproxy/letsencrypt:/etc/letsencrypt diff --git a/msa/black-box-tests/src/test/resources/docker-compose.redis-ssl.yml b/msa/black-box-tests/src/test/resources/docker-compose.redis-ssl.yml index e5444db329..0407aa3f8c 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.redis-ssl.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.redis-ssl.yml @@ -20,7 +20,7 @@ services: # Redis standalone redis: restart: always - image: bitnami/redis:7.0 + image: bitnami/redis:7.2 environment: # ALLOW_EMPTY_PASSWORD is recommended only for development. - 'ALLOW_EMPTY_PASSWORD=yes' From 0fbc89d89388986ad273b95d1f63bc2e1b4dbd5e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 25 Apr 2024 17:55:23 +0200 Subject: [PATCH 58/72] TbChangeOriginatorNodeTest refactored to use init() in @BeforeEach --- .../transform/TbChangeOriginatorNodeTest.java | 13 +++++-------- .../src/test/resources/junit-platform.properties | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java index 068858996b..952a566695 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNodeTest.java @@ -50,7 +50,6 @@ import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -//@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class) public class TbChangeOriginatorNodeTest { @@ -66,13 +65,13 @@ public class TbChangeOriginatorNodeTest { private ListeningExecutor dbExecutor; @BeforeEach - public void before() { + public void before() throws TbNodeException { dbExecutor = new TestDbCallbackExecutor(); + init(); } @Test - public void originatorCanBeChangedToCustomerId() throws TbNodeException { - init(); + public void originatorCanBeChangedToCustomerId() { AssetId assetId = new AssetId(Uuids.timeBased()); CustomerId customerId = new CustomerId(Uuids.timeBased()); Asset asset = new Asset(); @@ -96,8 +95,7 @@ public class TbChangeOriginatorNodeTest { } @Test - public void newChainCanBeStarted() throws TbNodeException { - init(); + public void newChainCanBeStarted() { AssetId assetId = new AssetId(Uuids.timeBased()); CustomerId customerId = new CustomerId(Uuids.timeBased()); Asset asset = new Asset(); @@ -120,8 +118,7 @@ public class TbChangeOriginatorNodeTest { } @Test - public void exceptionThrownIfCannotFindNewOriginator() throws TbNodeException { - init(); + public void exceptionThrownIfCannotFindNewOriginator() { AssetId assetId = new AssetId(Uuids.timeBased()); CustomerId customerId = new CustomerId(Uuids.timeBased()); Asset asset = new Asset(); diff --git a/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties b/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties index ad19ea833b..f2ed301920 100644 --- a/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties +++ b/rule-engine/rule-engine-components/src/test/resources/junit-platform.properties @@ -1,3 +1,3 @@ junit.jupiter.execution.parallel.enabled = true junit.jupiter.execution.parallel.mode.default = concurrent -junit.jupiter.execution.parallel.mode.classes.default = concurrent \ No newline at end of file +junit.jupiter.execution.parallel.mode.classes.default = concurrent From 086c63edb48ced1a64df58c86b1c3b61568f6569 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 25 Apr 2024 17:56:19 +0200 Subject: [PATCH 59/72] TbDeviceStateNodeTest refactored to avoid static mocks that fails some time due to concurrent test run --- .../engine/action/TbDeviceStateNodeTest.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java index 9352d64809..9b3d2dc29b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbDeviceStateNodeTest.java @@ -47,6 +47,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.tools.TbRateLimits; import java.util.UUID; +import java.util.function.BiConsumer; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -65,9 +66,9 @@ public class TbDeviceStateNodeTest { @Mock private TbContext ctxMock; @Mock - private static RuleEngineDeviceStateManager deviceStateManagerMock; + private RuleEngineDeviceStateManager deviceStateManagerMock; @Captor - private static ArgumentCaptor callbackCaptor; + private ArgumentCaptor callbackCaptor; private TbDeviceStateNode node; private TbDeviceStateNodeConfiguration config; @@ -246,7 +247,7 @@ public class TbDeviceStateNodeTest { @ParameterizedTest @MethodSource - public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, Runnable actionVerification) { + public void givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback(TbMsgType supportedEventType, BiConsumer> actionVerification) { // GIVEN given(ctxMock.getTenantId()).willReturn(TENANT_ID); given(ctxMock.getDeviceStateNodeRateLimitConfig()).willReturn("1:1"); @@ -262,7 +263,7 @@ public class TbDeviceStateNodeTest { node.onMsg(ctxMock, msg); // THEN - actionVerification.run(); + actionVerification.accept(this.deviceStateManagerMock, this.callbackCaptor); TbCallback actualCallback = callbackCaptor.getValue(); @@ -280,10 +281,10 @@ public class TbDeviceStateNodeTest { private static Stream givenSupportedEventAndDeviceOriginator_whenOnMsg_thenCorrectEventIsSentWithCorrectCallback() { return Stream.of( - Arguments.of(TbMsgType.CONNECT_EVENT, (Runnable) () -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.ACTIVITY_EVENT, (Runnable) () -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.DISCONNECT_EVENT, (Runnable) () -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), - Arguments.of(TbMsgType.INACTIVITY_EVENT, (Runnable) () -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) + Arguments.of(TbMsgType.CONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceConnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.ACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceActivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.DISCONNECT_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceDisconnect(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())), + Arguments.of(TbMsgType.INACTIVITY_EVENT, (BiConsumer>) (deviceStateManagerMock, callbackCaptor) -> then(deviceStateManagerMock).should().onDeviceInactivity(eq(TENANT_ID), eq(DEVICE_ID), eq(METADATA_TS), callbackCaptor.capture())) ); } From d12c680e121d331cdac36ea3ad2122cc7e8eaca8 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 25 Apr 2024 19:10:48 +0300 Subject: [PATCH 60/72] UI: Implement bars and polar area widgets. --- .../json/system/widget_bundles/charts.json | 16 +- .../data/json/system/widget_types/bars.json | 30 +- .../system/widget_types/bars_deprecated.json | 29 + .../json/system/widget_types/polar_area.json | 37 +- .../widget_types/polar_area_deprecated.json | 25 + .../basic/basic-widget-config.module.ts | 18 +- .../bar-chart-basic-config.component.html | 261 +++++++ .../chart/bar-chart-basic-config.component.ts | 316 ++++++++ ...rt-with-labels-basic-config.component.html | 18 +- .../doughnut-basic-config.component.html | 4 +- .../pie-chart-basic-config.component.html | 4 +- .../chart/pie-chart-basic-config.component.ts | 2 +- ...lar-area-chart-basic-config.component.html | 268 +++++++ ...polar-area-chart-basic-config.component.ts | 318 ++++++++ .../range-chart-basic-config.component.html | 12 +- .../range-chart-basic-config.component.ts | 12 +- ...e-series-chart-basic-config.component.html | 4 +- ...ime-series-chart-basic-config.component.ts | 4 +- .../lib/chart/bar-chart-widget.component.ts | 75 ++ .../lib/chart/bar-chart-widget.models.ts | 78 ++ .../bar-chart-with-labels-widget.models.ts | 26 +- .../widget/lib/chart/bars-chart.models.ts | 61 ++ .../components/widget/lib/chart/bars-chart.ts | 173 +++++ .../widget/lib/chart/chart.models.ts | 316 ++++++++ .../lib/chart/doughnut-widget.models.ts | 61 +- .../widget/lib/chart/echarts-widget.models.ts | 484 +----------- .../widget/lib/chart/latest-chart.models.ts | 67 +- .../lib/chart/pie-chart-widget.models.ts | 64 +- .../widget/lib/chart/pie-chart.models.ts | 67 +- .../lib/chart/polar-area-widget.component.ts | 75 ++ .../lib/chart/polar-area-widget.models.ts | 82 +++ .../lib/chart/range-chart-widget.models.ts | 48 +- .../lib/chart/time-series-chart-bar.models.ts | 5 +- .../chart/time-series-chart-state.models.ts | 6 +- .../time-series-chart-widget.component.ts | 5 +- .../chart/time-series-chart-widget.models.ts | 5 +- .../lib/chart/time-series-chart.models.ts | 688 ++++++++++++------ .../widget/lib/chart/time-series-chart.ts | 44 +- .../bar-chart-widget-settings.component.html | 169 +++++ .../bar-chart-widget-settings.component.ts | 160 ++++ ...with-labels-widget-settings.component.html | 18 +- .../doughnut-widget-settings.component.html | 4 +- .../pie-chart-widget-settings.component.html | 4 +- .../pie-chart-widget-settings.component.ts | 2 +- ...-area-chart-widget-settings.component.html | 176 +++++ ...ar-area-chart-widget-settings.component.ts | 161 ++++ ...range-chart-widget-settings.component.html | 12 +- .../range-chart-widget-settings.component.ts | 14 +- ...e-series-chart-key-settings.component.html | 6 +- ...-series-chart-line-settings.component.html | 12 +- ...me-series-chart-line-settings.component.ts | 17 +- ...eries-chart-widget-settings.component.html | 4 +- ...-series-chart-widget-settings.component.ts | 4 +- ...> chart-animation-settings.component.html} | 20 +- ... => chart-animation-settings.component.ts} | 23 +- .../chart/chart-bar-settings.component.html} | 31 +- .../chart/chart-bar-settings.component.ts} | 72 +- ...tml => chart-fill-settings.component.html} | 14 +- ...nt.ts => chart-fill-settings.component.ts} | 52 +- ...-series-chart-axis-settings.component.html | 14 +- ...rt-threshold-settings-panel.component.html | 8 +- ...hart-threshold-settings-panel.component.ts | 18 +- ...ime-series-chart-y-axis-row.component.html | 4 +- .../common/widget-settings-common.module.ts | 19 +- .../lib/settings/widget-settings.module.ts | 23 +- .../widget/widget-components.module.ts | 10 +- .../assets/locale/locale.constant-en_US.json | 106 +-- 67 files changed, 3738 insertions(+), 1247 deletions(-) create mode 100644 application/src/main/data/json/system/widget_types/bars_deprecated.json create mode 100644 application/src/main/data/json/system/widget_types/polar_area_deprecated.json create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-basic-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-basic-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.models.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.models.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/chart.models.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.models.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.ts rename ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/{echarts-animation-settings.component.html => chart-animation-settings.component.html} (82%) rename ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/{echarts-animation-settings.component.ts => chart-animation-settings.component.ts} (81%) rename ui-ngx/src/app/modules/home/components/widget/lib/settings/{chart/time-series-chart-bar-settings.component.html => common/chart/chart-bar-settings.component.html} (70%) rename ui-ngx/src/app/modules/home/components/widget/lib/settings/{chart/time-series-chart-bar-settings.component.ts => common/chart/chart-bar-settings.component.ts} (67%) rename ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/{time-series-chart-fill-settings.component.html => chart-fill-settings.component.html} (78%) rename ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/{time-series-chart-fill-settings.component.ts => chart-fill-settings.component.ts} (70%) diff --git a/application/src/main/data/json/system/widget_bundles/charts.json b/application/src/main/data/json/system/widget_bundles/charts.json index b7b1d643e3..01e575ea9b 100644 --- a/application/src/main/data/json/system/widget_bundles/charts.json +++ b/application/src/main/data/json/system/widget_bundles/charts.json @@ -15,18 +15,20 @@ "state_chart", "bar_chart_with_labels", "range_chart", + "cards.aggregated_value_card", + "bars", + "pie", + "doughnut", + "horizontal_doughnut", + "polar_area", + "charts.radar_chart_js", "charts.basic_timeseries", "charts.state_chart", "charts.timeseries_bars_flot", - "cards.aggregated_value_card", "charts.bars", - "pie", "charts.pie", "charts.pie_chart_js", - "doughnut", - "horizontal_doughnut", - "charts.polar_area_chart_js", - "charts.radar_chart_js", - "charts.doughnut_chart_js" + "charts.doughnut_chart_js", + "charts.polar_area_chart_js" ] } \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_types/bars.json b/application/src/main/data/json/system/widget_types/bars.json index b3c991f968..69c9ae208a 100644 --- a/application/src/main/data/json/system/widget_types/bars.json +++ b/application/src/main/data/json/system/widget_types/bars.json @@ -1,28 +1,26 @@ { - "fqn": "charts.bars", + "fqn": "bars", "name": "Bars", "deprecated": false, - "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAMAAAB+IdObAAAA8FBMVEUhlvNMr1Bqamp5eXl7e3t8fHx9fX1+fn5/f3+AgICCgoKDg4OEhISGhoaHh4eKioqMjIyNjY2Ojo6QkJCRkZGSkpKWlpaXl5ebm5udnZ2enp6goKChoaGkpKSnp6epqamsrKyurq6xsbGzs7O1tbW2tra3t7e4uLi7u7u9vb3BwcHCwsLDw8PGxsbKysrNzc3Ozs7R0dHS0tLT09PZ2dna2trc3Nzd3d3e3t7g4ODh4eHj4+Pk5OTm5ubn5+fo6Ojp6enu7u7w8PDz8/P0Qzb09PT29vb39/f5+fn6+vr7+/v8/Pz9/f3+/v7/wQf///+dc+aLAAAAAWJLR0RPbmZBSQAAAcFJREFUeNrt3ds2AgEYhuHsaSOZbAvZi0r2YYjCJOW7/7txZhkcDNbM6h/vdwfPmlX/ybtmEorJErGCeJLadz3rkKPpZamaLTp925DHdFvSpKelU9uQ/cLKhtcdk7YqtiHruevtojch7ZZtQ0o1dcdfRqXNqm1IbVU3OaVamm/YhvQW5zIXOknnC5JUt7qEpE5fUv/5HVePy2UHAgQIECBAgAABAgQIECBxgrwGHBAgQIAAAQIECBAgQIAAAQIECJC/QRIBN0iQ+66voDMLuRp2fQWdVUhvNun6CjqrkJ0Dx/UVdEYhzXzfcX0FnVFIrlSZ2mx/LOiMQuqHh6k972NBZ/fv13G/KeiCQkIu4358EL8UdBafSGwuOxAgQIAAAQIECJDYQB4CDggQIECAAAECBAgQIECAAAECBAgQIECA/BrSufn0DjqjkEZmLXkWaUEXEmThXMeFSAu68H4j5b1IC7rQILfZTqQFXViQ1nRTL1EWdCFBnmYuJUVZ0IUEWR1xHKcXZUEXFDLwBR2XHQgQIECAAAEC5H9ChgIOCBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgxiBmv+L6Bl9pkxYph15gAAAAAElFTkSuQmCC", - "description": "Displays latest values of the attributes or time-series data for multiple entities as separate bars.", + "image": "tb-image:YmFycy5zdmc=:IkJhcnMiIHN5c3RlbSB3aWRnZXQgaW1hZ2U=;data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAwIiBoZWlnaHQ9IjE2MCIgdmlld0JveD0iMCAwIDIwMCAxNjAiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxnIGNsaXAtcGF0aD0idXJsKCNjbGlwMF80NjI0XzM4MDIyKSI+CjxwYXRoIGQ9Ik0yLjg0NzY2IDEuMjgxMjVWN0gyLjEyNVYyLjE4MzU5TDAuNjY3OTY5IDIuNzE0ODRWMi4wNjI1TDIuNzM0MzggMS4yODEyNUgyLjg0NzY2Wk04LjY3NTE3IDMuNzAzMTJWNC41NzAzMUM4LjY3NTE3IDUuMDM2NDYgOC42MzM1MSA1LjQyOTY5IDguNTUwMTcgNS43NUM4LjQ2Njg0IDYuMDcwMzEgOC4zNDcwNSA2LjMyODEyIDguMTkwOCA2LjUyMzQ0QzguMDM0NTUgNi43MTg3NSA3Ljg0NTc1IDYuODYwNjggNy42MjQzOSA2Ljk0OTIyQzcuNDA1NjQgNy4wMzUxNiA3LjE1ODI1IDcuMDc4MTIgNi44ODIyIDcuMDc4MTJDNi42NjM0NSA3LjA3ODEyIDYuNDYxNjMgNy4wNTA3OCA2LjI3NjczIDYuOTk2MDlDNi4wOTE4NCA2Ljk0MTQxIDUuOTI1MTcgNi44NTQxNyA1Ljc3NjczIDYuNzM0MzhDNS42MzA5IDYuNjExOTggNS41MDU5IDYuNDUzMTIgNS40MDE3MyA2LjI1NzgxQzUuMjk3NTcgNi4wNjI1IDUuMjE4MTQgNS44MjU1MiA1LjE2MzQ1IDUuNTQ2ODhDNS4xMDg3NyA1LjI2ODIzIDUuMDgxNDIgNC45NDI3MSA1LjA4MTQyIDQuNTcwMzFWMy43MDMxMkM1LjA4MTQyIDMuMjM2OTggNS4xMjMwOSAyLjg0NjM1IDUuMjA2NDIgMi41MzEyNUM1LjI5MjM2IDIuMjE2MTUgNS40MTM0NSAxLjk2MzU0IDUuNTY5NyAxLjc3MzQ0QzUuNzI1OTUgMS41ODA3MyA1LjkxMzQ1IDEuNDQyNzEgNi4xMzIyIDEuMzU5MzhDNi4zNTM1NiAxLjI3NjA0IDYuNjAwOTUgMS4yMzQzOCA2Ljg3NDM5IDEuMjM0MzhDNy4wOTU3NSAxLjIzNDM4IDcuMjk4ODcgMS4yNjE3MiA3LjQ4Mzc3IDEuMzE2NDFDNy42NzEyNyAxLjM2ODQ5IDcuODM3OTMgMS40NTMxMiA3Ljk4Mzc3IDEuNTcwMzFDOC4xMjk2IDEuNjg0OSA4LjI1MzMgMS44Mzg1NCA4LjM1NDg2IDIuMDMxMjVDOC40NTkwMyAyLjIyMTM1IDguNTM4NDUgMi40NTQ0MyA4LjU5MzE0IDIuNzMwNDdDOC42NDc4MyAzLjAwNjUxIDguNjc1MTcgMy4zMzA3MyA4LjY3NTE3IDMuNzAzMTJaTTcuOTQ4NjEgNC42ODc1VjMuNTgyMDNDNy45NDg2MSAzLjMyNjgyIDcuOTMyOTggMy4xMDI4NiA3LjkwMTczIDIuOTEwMTZDNy44NzMwOSAyLjcxNDg0IDcuODMwMTIgMi41NDgxOCA3Ljc3MjgzIDIuNDEwMTZDNy43MTU1NCAyLjI3MjE0IDcuNjQyNjIgMi4xNjAxNiA3LjU1NDA4IDIuMDc0MjJDNy40NjgxNCAxLjk4ODI4IDcuMzY3ODggMS45MjU3OCA3LjI1MzMgMS44ODY3MkM3LjE0MTMyIDEuODQ1MDUgNy4wMTUwMiAxLjgyNDIyIDYuODc0MzkgMS44MjQyMkM2LjcwMjUyIDEuODI0MjIgNi41NTAxNyAxLjg1Njc3IDYuNDE3MzYgMS45MjE4OEM2LjI4NDU1IDEuOTg0MzggNi4xNzI1NyAyLjA4NDY0IDYuMDgxNDIgMi4yMjI2NkM1Ljk5Mjg4IDIuMzYwNjggNS45MjUxNyAyLjU0MTY3IDUuODc4MyAyLjc2NTYyQzUuODMxNDIgMi45ODk1OCA1LjgwNzk4IDMuMjYxNzIgNS44MDc5OCAzLjU4MjAzVjQuNjg3NUM1LjgwNzk4IDQuOTQyNzEgNS44MjIzMSA1LjE2Nzk3IDUuODUwOTUgNS4zNjMyOEM1Ljg4MjIgNS41NTg1OSA1LjkyNzc4IDUuNzI3ODYgNS45ODc2NyA1Ljg3MTA5QzYuMDQ3NTcgNi4wMTE3MiA2LjEyMDQ4IDYuMTI3NiA2LjIwNjQyIDYuMjE4NzVDNi4yOTIzNiA2LjMwOTkgNi4zOTEzMiA2LjM3NzYgNi41MDMzIDYuNDIxODhDNi42MTc4OCA2LjQ2MzU0IDYuNzQ0MTggNi40ODQzOCA2Ljg4MjIgNi40ODQzOEM3LjA1OTI5IDYuNDg0MzggNy4yMTQyMyA2LjQ1MDUyIDcuMzQ3MDUgNi4zODI4MUM3LjQ3OTg2IDYuMzE1MSA3LjU5MDU0IDYuMjA5NjQgNy42NzkwOCA2LjA2NjQxQzcuNzcwMjIgNS45MjA1NyA3LjgzNzkzIDUuNzM0MzggNy44ODIyIDUuNTA3ODFDNy45MjY0NyA1LjI3ODY1IDcuOTQ4NjEgNS4wMDUyMSA3Ljk0ODYxIDQuNjg3NVpNMTMuMzA3NCAzLjcwMzEyVjQuNTcwMzFDMTMuMzA3NCA1LjAzNjQ2IDEzLjI2NTcgNS40Mjk2OSAxMy4xODI0IDUuNzVDMTMuMDk5IDYuMDcwMzEgMTIuOTc5MyA2LjMyODEyIDEyLjgyMyA2LjUyMzQ0QzEyLjY2NjggNi43MTg3NSAxMi40Nzc5IDYuODYwNjggMTIuMjU2NiA2Ljk0OTIyQzEyLjAzNzggNy4wMzUxNiAxMS43OTA0IDcuMDc4MTIgMTEuNTE0NCA3LjA3ODEyQzExLjI5NTcgNy4wNzgxMiAxMS4wOTM4IDcuMDUwNzggMTAuOTA4OSA2Ljk5NjA5QzEwLjcyNCA2Ljk0MTQxIDEwLjU1NzQgNi44NTQxNyAxMC40MDg5IDYuNzM0MzhDMTAuMjYzMSA2LjYxMTk4IDEwLjEzODEgNi40NTMxMiAxMC4wMzM5IDYuMjU3ODFDOS45Mjk3NyA2LjA2MjUgOS44NTAzNCA1LjgyNTUyIDkuNzk1NjYgNS41NDY4OEM5Ljc0MDk3IDUuMjY4MjMgOS43MTM2MyA0Ljk0MjcxIDkuNzEzNjMgNC41NzAzMVYzLjcwMzEyQzkuNzEzNjMgMy4yMzY5OCA5Ljc1NTI5IDIuODQ2MzUgOS44Mzg2MyAyLjUzMTI1QzkuOTI0NTYgMi4yMTYxNSAxMC4wNDU3IDEuOTYzNTQgMTAuMjAxOSAxLjc3MzQ0QzEwLjM1ODIgMS41ODA3MyAxMC41NDU3IDEuNDQyNzEgMTAuNzY0NCAxLjM1OTM4QzEwLjk4NTggMS4yNzYwNCAxMS4yMzMyIDEuMjM0MzggMTEuNTA2NiAxLjIzNDM4QzExLjcyNzkgMS4yMzQzOCAxMS45MzExIDEuMjYxNzIgMTIuMTE2IDEuMzE2NDFDMTIuMzAzNSAxLjM2ODQ5IDEyLjQ3MDEgMS40NTMxMiAxMi42MTYgMS41NzAzMUMxMi43NjE4IDEuNjg0OSAxMi44ODU1IDEuODM4NTQgMTIuOTg3MSAyLjAzMTI1QzEzLjA5MTIgMi4yMjEzNSAxMy4xNzA3IDIuNDU0NDMgMTMuMjI1MyAyLjczMDQ3QzEzLjI4IDMuMDA2NTEgMTMuMzA3NCAzLjMzMDczIDEzLjMwNzQgMy43MDMxMlpNMTIuNTgwOCA0LjY4NzVWMy41ODIwM0MxMi41ODA4IDMuMzI2ODIgMTIuNTY1MiAzLjEwMjg2IDEyLjUzMzkgMi45MTAxNkMxMi41MDUzIDIuNzE0ODQgMTIuNDYyMyAyLjU0ODE4IDEyLjQwNSAyLjQxMDE2QzEyLjM0NzcgMi4yNzIxNCAxMi4yNzQ4IDIuMTYwMTYgMTIuMTg2MyAyLjA3NDIyQzEyLjEwMDMgMS45ODgyOCAxMi4wMDAxIDEuOTI1NzggMTEuODg1NSAxLjg4NjcyQzExLjc3MzUgMS44NDUwNSAxMS42NDcyIDEuODI0MjIgMTEuNTA2NiAxLjgyNDIyQzExLjMzNDcgMS44MjQyMiAxMS4xODI0IDEuODU2NzcgMTEuMDQ5NiAxLjkyMTg4QzEwLjkxNjggMS45ODQzOCAxMC44MDQ4IDIuMDg0NjQgMTAuNzEzNiAyLjIyMjY2QzEwLjYyNTEgMi4zNjA2OCAxMC41NTc0IDIuNTQxNjcgMTAuNTEwNSAyLjc2NTYyQzEwLjQ2MzYgMi45ODk1OCAxMC40NDAyIDMuMjYxNzIgMTAuNDQwMiAzLjU4MjAzVjQuNjg3NUMxMC40NDAyIDQuOTQyNzEgMTAuNDU0NSA1LjE2Nzk3IDEwLjQ4MzIgNS4zNjMyOEMxMC41MTQ0IDUuNTU4NTkgMTAuNTYgNS43Mjc4NiAxMC42MTk5IDUuODcxMDlDMTAuNjc5OCA2LjAxMTcyIDEwLjc1MjcgNi4xMjc2IDEwLjgzODYgNi4yMTg3NUMxMC45MjQ2IDYuMzA5OSAxMS4wMjM1IDYuMzc3NiAxMS4xMzU1IDYuNDIxODhDMTEuMjUwMSA2LjQ2MzU0IDExLjM3NjQgNi40ODQzOCAxMS41MTQ0IDYuNDg0MzhDMTEuNjkxNSA2LjQ4NDM4IDExLjg0NjQgNi40NTA1MiAxMS45NzkzIDYuMzgyODFDMTIuMTEyMSA2LjMxNTEgMTIuMjIyNyA2LjIwOTY0IDEyLjMxMTMgNi4wNjY0MUMxMi40MDI0IDUuOTIwNTcgMTIuNDcwMSA1LjczNDM4IDEyLjUxNDQgNS41MDc4MUMxMi41NTg3IDUuMjc4NjUgMTIuNTgwOCA1LjAwNTIxIDEyLjU4MDggNC42ODc1WiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8cGF0aCBkPSJNOC4wNTg1OSAzMy42Njc3QzguMDU4NTkgMzQuMDE0MSA3Ljk3Nzg2IDM0LjMwODMgNy44MTY0MSAzNC41NTA1QzcuNjU3NTUgMzQuNzkwMSA3LjQ0MTQxIDM0Ljk3MjQgNy4xNjc5NyAzNS4wOTc0QzYuODk3MTQgMzUuMjIyNCA2LjU5MTE1IDM1LjI4NDkgNi4yNSAzNS4yODQ5QzUuOTA4ODUgMzUuMjg0OSA1LjYwMTU2IDM1LjIyMjQgNS4zMjgxMiAzNS4wOTc0QzUuMDU0NjkgMzQuOTcyNCA0LjgzODU0IDM0Ljc5MDEgNC42Nzk2OSAzNC41NTA1QzQuNTIwODMgMzQuMzA4MyA0LjQ0MTQxIDM0LjAxNDEgNC40NDE0MSAzMy42Njc3QzQuNDQxNDEgMzMuNDQxMiA0LjQ4NDM4IDMzLjIzNDEgNC41NzAzMSAzMy4wNDY2QzQuNjU4ODUgMzIuODU2NSA0Ljc4MjU1IDMyLjY5MTIgNC45NDE0MSAzMi41NTA1QzUuMTAyODYgMzIuNDA5OSA1LjI5Mjk3IDMyLjMwMTggNS41MTE3MiAzMi4yMjYzQzUuNzMzMDcgMzIuMTQ4MiA1Ljk3NjU2IDMyLjEwOTEgNi4yNDIxOSAzMi4xMDkxQzYuNTkxMTUgMzIuMTA5MSA2LjkwMjM0IDMyLjE3NjggNy4xNzU3OCAzMi4zMTIzQzcuNDQ5MjIgMzIuNDQ1MSA3LjY2NDA2IDMyLjYyODcgNy44MjAzMSAzMi44NjNDNy45NzkxNyAzMy4wOTc0IDguMDU4NTkgMzMuMzY1NiA4LjA1ODU5IDMzLjY2NzdaTTcuMzMyMDMgMzMuNjUyMUM3LjMzMjAzIDMzLjQ0MTIgNy4yODY0NiAzMy4yNTUgNy4xOTUzMSAzMy4wOTM1QzcuMTA0MTcgMzIuOTI5NCA2Ljk3NjU2IDMyLjgwMTggNi44MTI1IDMyLjcxMDdDNi42NDg0NCAzMi42MTk1IDYuNDU4MzMgMzIuNTc0IDYuMjQyMTkgMzIuNTc0QzYuMDIwODMgMzIuNTc0IDUuODI5NDMgMzIuNjE5NSA1LjY2Nzk3IDMyLjcxMDdDNS41MDkxMSAzMi44MDE4IDUuMzg1NDIgMzIuOTI5NCA1LjI5Njg4IDMzLjA5MzVDNS4yMDgzMyAzMy4yNTUgNS4xNjQwNiAzMy40NDEyIDUuMTY0MDYgMzMuNjUyMUM1LjE2NDA2IDMzLjg3MDggNS4yMDcwMyAzNC4wNTgzIDUuMjkyOTcgMzQuMjE0NkM1LjM4MTUxIDM0LjM2ODIgNS41MDY1MSAzNC40ODY3IDUuNjY3OTcgMzQuNTcwMUM1LjgzMjAzIDM0LjY1MDggNi4wMjYwNCAzNC42OTEyIDYuMjUgMzQuNjkxMkM2LjQ3Mzk2IDM0LjY5MTIgNi42NjY2NyAzNC42NTA4IDYuODI4MTIgMzQuNTcwMUM2Ljk4OTU4IDM0LjQ4NjcgNy4xMTMyOCAzNC4zNjgyIDcuMTk5MjIgMzQuMjE0NkM3LjI4Nzc2IDM0LjA1ODMgNy4zMzIwMyAzMy44NzA4IDcuMzMyMDMgMzMuNjUyMVpNNy45MjU3OCAzMC45OTk4QzcuOTI1NzggMzEuMjc1OCA3Ljg1Mjg2IDMxLjUyNDUgNy43MDcwMyAzMS43NDU4QzcuNTYxMiAzMS45NjcyIDcuMzYxOTggMzIuMTQxNyA3LjEwOTM4IDMyLjI2OTNDNi44NTY3NyAzMi4zOTY5IDYuNTcwMzEgMzIuNDYwNyA2LjI1IDMyLjQ2MDdDNS45MjQ0OCAzMi40NjA3IDUuNjM0MTEgMzIuMzk2OSA1LjM3ODkxIDMyLjI2OTNDNS4xMjYzIDMyLjE0MTcgNC45MjgzOSAzMS45NjcyIDQuNzg1MTYgMzEuNzQ1OEM0LjY0MTkzIDMxLjUyNDUgNC41NzAzMSAzMS4yNzU4IDQuNTcwMzEgMzAuOTk5OEM0LjU3MDMxIDMwLjY2OSA0LjY0MTkzIDMwLjM4NzggNC43ODUxNiAzMC4xNTZDNC45MzA5OSAyOS45MjQyIDUuMTMwMjEgMjkuNzQ3MiA1LjM4MjgxIDI5LjYyNDhDNS42MzU0MiAyOS41MDI0IDUuOTIzMTggMjkuNDQxMiA2LjI0NjA5IDI5LjQ0MTJDNi41NzE2MSAyOS40NDEyIDYuODYwNjggMjkuNTAyNCA3LjExMzI4IDI5LjYyNDhDNy4zNjU4OSAyOS43NDcyIDcuNTYzOCAyOS45MjQyIDcuNzA3MDMgMzAuMTU2QzcuODUyODYgMzAuMzg3OCA3LjkyNTc4IDMwLjY2OSA3LjkyNTc4IDMwLjk5OThaTTcuMjAzMTIgMzEuMDExNUM3LjIwMzEyIDMwLjgyMTQgNy4xNjI3NiAzMC42NTM0IDcuMDgyMDMgMzAuNTA3NkM3LjAwMTMgMzAuMzYxNyA2Ljg4OTMyIDMwLjI0NzIgNi43NDYwOSAzMC4xNjM4QzYuNjAyODYgMzAuMDc3OSA2LjQzNjIgMzAuMDM0OSA2LjI0NjA5IDMwLjAzNDlDNi4wNTU5OSAzMC4wMzQ5IDUuODg5MzIgMzAuMDc1MyA1Ljc0NjA5IDMwLjE1NkM1LjYwNTQ3IDMwLjIzNDEgNS40OTQ3OSAzMC4zNDYxIDUuNDE0MDYgMzAuNDkxOUM1LjMzNTk0IDMwLjYzNzggNS4yOTY4OCAzMC44MTEgNS4yOTY4OCAzMS4wMTE1QzUuMjk2ODggMzEuMjA2OCA1LjMzNTk0IDMxLjM3NzQgNS40MTQwNiAzMS41MjMyQzUuNDk0NzkgMzEuNjY5IDUuNjA2NzcgMzEuNzgyMyA1Ljc1IDMxLjg2M0M1Ljg5MzIzIDMxLjk0MzggNi4wNTk5IDMxLjk4NDEgNi4yNSAzMS45ODQxQzYuNDQwMSAzMS45ODQxIDYuNjA1NDcgMzEuOTQzOCA2Ljc0NjA5IDMxLjg2M0M2Ljg4OTMyIDMxLjc4MjMgNy4wMDEzIDMxLjY2OSA3LjA4MjAzIDMxLjUyMzJDNy4xNjI3NiAzMS4zNzc0IDcuMjAzMTIgMzEuMjA2OCA3LjIwMzEyIDMxLjAxMTVaTTEyLjY3NTIgMzEuOTA5OVYzMi43NzcxQzEyLjY3NTIgMzMuMjQzMiAxMi42MzM1IDMzLjYzNjUgMTIuNTUwMiAzMy45NTY4QzEyLjQ2NjggMzQuMjc3MSAxMi4zNDcgMzQuNTM0OSAxMi4xOTA4IDM0LjczMDJDMTIuMDM0NSAzNC45MjU1IDExLjg0NTcgMzUuMDY3NSAxMS42MjQ0IDM1LjE1NkMxMS40MDU2IDM1LjI0MTkgMTEuMTU4MiAzNS4yODQ5IDEwLjg4MjIgMzUuMjg0OUMxMC42NjM1IDM1LjI4NDkgMTAuNDYxNiAzNS4yNTc2IDEwLjI3NjcgMzUuMjAyOUMxMC4wOTE4IDM1LjE0ODIgOS45MjUxNyAzNS4wNjEgOS43NzY3MyAzNC45NDEyQzkuNjMwOSAzNC44MTg4IDkuNTA1OSAzNC42NTk5IDkuNDAxNzMgMzQuNDY0NkM5LjI5NzU3IDM0LjI2OTMgOS4yMTgxNCAzNC4wMzIzIDkuMTYzNDUgMzMuNzUzN0M5LjEwODc3IDMzLjQ3NSA5LjA4MTQyIDMzLjE0OTUgOS4wODE0MiAzMi43NzcxVjMxLjkwOTlDOS4wODE0MiAzMS40NDM4IDkuMTIzMDkgMzEuMDUzMSA5LjIwNjQyIDMwLjczOEM5LjI5MjM2IDMwLjQyMjkgOS40MTM0NSAzMC4xNzAzIDkuNTY5NyAyOS45ODAyQzkuNzI1OTUgMjkuNzg3NSA5LjkxMzQ1IDI5LjY0OTUgMTAuMTMyMiAyOS41NjYyQzEwLjM1MzYgMjkuNDgyOCAxMC42MDEgMjkuNDQxMiAxMC44NzQ0IDI5LjQ0MTJDMTEuMDk1NyAyOS40NDEyIDExLjI5ODkgMjkuNDY4NSAxMS40ODM4IDI5LjUyMzJDMTEuNjcxMyAyOS41NzUzIDExLjgzNzkgMjkuNjU5OSAxMS45ODM4IDI5Ljc3NzFDMTIuMTI5NiAyOS44OTE3IDEyLjI1MzMgMzAuMDQ1MyAxMi4zNTQ5IDMwLjIzOEMxMi40NTkgMzAuNDI4MSAxMi41Mzg1IDMwLjY2MTIgMTIuNTkzMSAzMC45MzczQzEyLjY0NzggMzEuMjEzMyAxMi42NzUyIDMxLjUzNzUgMTIuNjc1MiAzMS45MDk5Wk0xMS45NDg2IDMyLjg5NDNWMzEuNzg4OEMxMS45NDg2IDMxLjUzMzYgMTEuOTMzIDMxLjMwOTcgMTEuOTAxNyAzMS4xMTY5QzExLjg3MzEgMzAuOTIxNiAxMS44MzAxIDMwLjc1NSAxMS43NzI4IDMwLjYxNjlDMTEuNzE1NSAzMC40Nzg5IDExLjY0MjYgMzAuMzY2OSAxMS41NTQxIDMwLjI4MUMxMS40NjgxIDMwLjE5NTEgMTEuMzY3OSAzMC4xMzI2IDExLjI1MzMgMzAuMDkzNUMxMS4xNDEzIDMwLjA1MTggMTEuMDE1IDMwLjAzMSAxMC44NzQ0IDMwLjAzMUMxMC43MDI1IDMwLjAzMSAxMC41NTAyIDMwLjA2MzYgMTAuNDE3NCAzMC4xMjg3QzEwLjI4NDUgMzAuMTkxMiAxMC4xNzI2IDMwLjI5MTQgMTAuMDgxNCAzMC40Mjk0QzkuOTkyODggMzAuNTY3NSA5LjkyNTE3IDMwLjc0ODUgOS44NzgzIDMwLjk3MjRDOS44MzE0MiAzMS4xOTY0IDkuODA3OTggMzEuNDY4NSA5LjgwNzk4IDMxLjc4ODhWMzIuODk0M0M5LjgwNzk4IDMzLjE0OTUgOS44MjIzMSAzMy4zNzQ4IDkuODUwOTUgMzMuNTcwMUM5Ljg4MjIgMzMuNzY1NCA5LjkyNzc4IDMzLjkzNDcgOS45ODc2NyAzNC4wNzc5QzEwLjA0NzYgMzQuMjE4NSAxMC4xMjA1IDM0LjMzNDQgMTAuMjA2NCAzNC40MjU1QzEwLjI5MjQgMzQuNTE2NyAxMC4zOTEzIDM0LjU4NDQgMTAuNTAzMyAzNC42Mjg3QzEwLjYxNzkgMzQuNjcwMyAxMC43NDQyIDM0LjY5MTIgMTAuODgyMiAzNC42OTEyQzExLjA1OTMgMzQuNjkxMiAxMS4yMTQyIDM0LjY1NzMgMTEuMzQ3IDM0LjU4OTZDMTEuNDc5OSAzNC41MjE5IDExLjU5MDUgMzQuNDE2NCAxMS42NzkxIDM0LjI3MzJDMTEuNzcwMiAzNC4xMjc0IDExLjgzNzkgMzMuOTQxMiAxMS44ODIyIDMzLjcxNDZDMTEuOTI2NSAzMy40ODU0IDExLjk0ODYgMzMuMjEyIDExLjk0ODYgMzIuODk0M1oiIGZpbGw9ImJsYWNrIiBmaWxsLW9wYWNpdHk9IjAuNzYiLz4KPHBhdGggZD0iTTcuMjQ2MDkgNTcuNzE4M0g3LjMwODU5VjU4LjMzMTVINy4yNDYwOUM2Ljg2MzI4IDU4LjMzMTUgNi41NDI5NyA1OC4zOTQgNi4yODUxNiA1OC41MTlDNi4wMjczNCA1OC42NDE0IDUuODIyOTIgNTguODA2OCA1LjY3MTg4IDU5LjAxNTFDNS41MjA4MyA1OS4yMjA5IDUuNDExNDYgNTkuNDUyNiA1LjM0Mzc1IDU5LjcxMDRDNS4yNzg2NSA1OS45NjgzIDUuMjQ2MDkgNjAuMjMgNS4yNDYwOSA2MC40OTU2VjYxLjMzMTVDNS4yNDYwOSA2MS41ODQxIDUuMjc2MDQgNjEuODA4MSA1LjMzNTk0IDYyLjAwMzRDNS4zOTU4MyA2Mi4xOTYxIDUuNDc3ODYgNjIuMzU4OSA1LjU4MjAzIDYyLjQ5MTdDNS42ODYyIDYyLjYyNDUgNS44MDMzOSA2Mi43MjQ4IDUuOTMzNTkgNjIuNzkyNUM2LjA2NjQxIDYyLjg2MDIgNi4yMDQ0MyA2Mi44OTQgNi4zNDc2NiA2Mi44OTRDNi41MTQzMiA2Mi44OTQgNi42NjI3NiA2Mi44NjI4IDYuNzkyOTcgNjIuODAwM0M2LjkyMzE4IDYyLjczNTIgNy4wMzI1NSA2Mi42NDUzIDcuMTIxMDkgNjIuNTMwOEM3LjIxMjI0IDYyLjQxMzYgNy4yODEyNSA2Mi4yNzU2IDcuMzI4MTIgNjIuMTE2N0M3LjM3NSA2MS45NTc4IDcuMzk4NDQgNjEuNzgzNCA3LjM5ODQ0IDYxLjU5MzNDNy4zOTg0NCA2MS40MjQgNy4zNzc2IDYxLjI2MTIgNy4zMzU5NCA2MS4xMDVDNy4yOTQyNyA2MC45NDYxIDcuMjMwNDcgNjAuODA1NSA3LjE0NDUzIDYwLjY4MzFDNy4wNTg1OSA2MC41NTgxIDYuOTUwNTIgNjAuNDYwNCA2LjgyMDMxIDYwLjM5MDFDNi42OTI3MSA2MC4zMTcyIDYuNTQwMzYgNjAuMjgwOCA2LjM2MzI4IDYwLjI4MDhDNi4xNjI3NiA2MC4yODA4IDUuOTc1MjYgNjAuMzMwMiA1LjgwMDc4IDYwLjQyOTJDNS42Mjg5MSA2MC41MjU2IDUuNDg2OTggNjAuNjUzMiA1LjM3NSA2MC44MTJDNS4yNjU2MiA2MC45NjgzIDUuMjAzMTIgNjEuMTM4OCA1LjE4NzUgNjEuMzIzN0w0LjgwNDY5IDYxLjMxOThDNC44NDExNSA2MS4wMjgyIDQuOTA4ODUgNjAuNzc5NSA1LjAwNzgxIDYwLjU3MzdDNS4xMDkzOCA2MC4zNjU0IDUuMjM0MzggNjAuMTk2MSA1LjM4MjgxIDYwLjA2NTlDNS41MzM4NSA1OS45MzMxIDUuNzAxODIgNTkuODM2OCA1Ljg4NjcyIDU5Ljc3NjlDNi4wNzQyMiA1OS43MTQ0IDYuMjcyMTQgNTkuNjgzMSA2LjQ4MDQ3IDU5LjY4MzFDNi43NjQzMiA1OS42ODMxIDcuMDA5MTEgNTkuNzM2NSA3LjIxNDg0IDU5Ljg0MzNDNy40MjA1NyA1OS45NSA3LjU4OTg0IDYwLjA5MzMgNy43MjI2NiA2MC4yNzI5QzcuODU1NDcgNjAuNDUgNy45NTMxMiA2MC42NTA2IDguMDE1NjIgNjAuODc0NUM4LjA4MDczIDYxLjA5NTkgOC4xMTMyOCA2MS4zMjM3IDguMTEzMjggNjEuNTU4MUM4LjExMzI4IDYxLjgyNjMgOC4wNzU1MiA2Mi4wNzc2IDggNjIuMzEyQzcuOTI0NDggNjIuNTQ2NCA3LjgxMTIgNjIuNzUyMSA3LjY2MDE2IDYyLjkyOTJDNy41MTE3MiA2My4xMDYzIDcuMzI4MTIgNjMuMjQ0MyA3LjEwOTM4IDYzLjM0MzNDNi44OTA2MiA2My40NDIyIDYuNjM2NzIgNjMuNDkxNyA2LjM0NzY2IDYzLjQ5MTdDNi4wNDAzNiA2My40OTE3IDUuNzcyMTQgNjMuNDI5MiA1LjU0Mjk3IDYzLjMwNDJDNS4zMTM4IDYzLjE3NjYgNS4xMjM3IDYzLjAwNzMgNC45NzI2NiA2Mi43OTY0QzQuODIxNjEgNjIuNTg1NCA0LjcwODMzIDYyLjM1MTEgNC42MzI4MSA2Mi4wOTMzQzQuNTU3MjkgNjEuODM1NCA0LjUxOTUzIDYxLjU3MzcgNC41MTk1MyA2MS4zMDgxVjYwLjk2ODNDNC41MTk1MyA2MC41NjcyIDQuNTU5OSA2MC4xNzQgNC42NDA2MiA1OS43ODg2QzQuNzIxMzUgNTkuNDAzMiA0Ljg2MDY4IDU5LjA1NDIgNS4wNTg1OSA1OC43NDE3QzUuMjU5MTEgNTguNDI5MiA1LjUzNjQ2IDU4LjE4MDUgNS44OTA2MiA1Ny45OTU2QzYuMjQ0NzkgNTcuODEwNyA2LjY5NjYxIDU3LjcxODMgNy4yNDYwOSA1Ny43MTgzWk0xMi42NzUyIDYwLjExNjdWNjAuOTgzOUMxMi42NzUyIDYxLjQ1IDEyLjYzMzUgNjEuODQzMyAxMi41NTAyIDYyLjE2MzZDMTIuNDY2OCA2Mi40ODM5IDEyLjM0NyA2Mi43NDE3IDEyLjE5MDggNjIuOTM3QzEyLjAzNDUgNjMuMTMyMyAxMS44NDU3IDYzLjI3NDMgMTEuNjI0NCA2My4zNjI4QzExLjQwNTYgNjMuNDQ4NyAxMS4xNTgyIDYzLjQ5MTcgMTAuODgyMiA2My40OTE3QzEwLjY2MzUgNjMuNDkxNyAxMC40NjE2IDYzLjQ2NDQgMTAuMjc2NyA2My40MDk3QzEwLjA5MTggNjMuMzU1IDkuOTI1MTcgNjMuMjY3NyA5Ljc3NjczIDYzLjE0NzlDOS42MzA5IDYzLjAyNTYgOS41MDU5IDYyLjg2NjcgOS40MDE3MyA2Mi42NzE0QzkuMjk3NTcgNjIuNDc2MSA5LjIxODE0IDYyLjIzOTEgOS4xNjM0NSA2MS45NjA0QzkuMTA4NzcgNjEuNjgxOCA5LjA4MTQyIDYxLjM1NjMgOS4wODE0MiA2MC45ODM5VjYwLjExNjdDOS4wODE0MiA1OS42NTA2IDkuMTIzMDkgNTkuMjU5OSA5LjIwNjQyIDU4Ljk0NDhDOS4yOTIzNiA1OC42Mjk3IDkuNDEzNDUgNTguMzc3MSA5LjU2OTcgNTguMTg3QzkuNzI1OTUgNTcuOTk0MyA5LjkxMzQ1IDU3Ljg1NjMgMTAuMTMyMiA1Ny43NzI5QzEwLjM1MzYgNTcuNjg5NiAxMC42MDEgNTcuNjQ3OSAxMC44NzQ0IDU3LjY0NzlDMTEuMDk1NyA1Ny42NDc5IDExLjI5ODkgNTcuNjc1MyAxMS40ODM4IDU3LjczQzExLjY3MTMgNTcuNzgyMSAxMS44Mzc5IDU3Ljg2NjcgMTEuOTgzOCA1Ny45ODM5QzEyLjEyOTYgNTguMDk4NSAxMi4yNTMzIDU4LjI1MjEgMTIuMzU0OSA1OC40NDQ4QzEyLjQ1OSA1OC42MzQ5IDEyLjUzODUgNTguODY4IDEyLjU5MzEgNTkuMTQ0QzEyLjY0NzggNTkuNDIwMSAxMi42NzUyIDU5Ljc0NDMgMTIuNjc1MiA2MC4xMTY3Wk0xMS45NDg2IDYxLjEwMTFWNTkuOTk1NkMxMS45NDg2IDU5Ljc0MDQgMTEuOTMzIDU5LjUxNjQgMTEuOTAxNyA1OS4zMjM3QzExLjg3MzEgNTkuMTI4NCAxMS44MzAxIDU4Ljk2MTggMTEuNzcyOCA1OC44MjM3QzExLjcxNTUgNTguNjg1NyAxMS42NDI2IDU4LjU3MzcgMTEuNTU0MSA1OC40ODc4QzExLjQ2ODEgNTguNDAxOSAxMS4zNjc5IDU4LjMzOTQgMTEuMjUzMyA1OC4zMDAzQzExLjE0MTMgNTguMjU4NiAxMS4wMTUgNTguMjM3OCAxMC44NzQ0IDU4LjIzNzhDMTAuNzAyNSA1OC4yMzc4IDEwLjU1MDIgNTguMjcwMyAxMC40MTc0IDU4LjMzNTRDMTAuMjg0NSA1OC4zOTc5IDEwLjE3MjYgNTguNDk4MiAxMC4wODE0IDU4LjYzNjJDOS45OTI4OCA1OC43NzQzIDkuOTI1MTcgNTguOTU1MiA5Ljg3ODMgNTkuMTc5MkM5LjgzMTQyIDU5LjQwMzIgOS44MDc5OCA1OS42NzUzIDkuODA3OTggNTkuOTk1NlY2MS4xMDExQzkuODA3OTggNjEuMzU2MyA5LjgyMjMxIDYxLjU4MTUgOS44NTA5NSA2MS43NzY5QzkuODgyMiA2MS45NzIyIDkuOTI3NzggNjIuMTQxNCA5Ljk4NzY3IDYyLjI4NDdDMTAuMDQ3NiA2Mi40MjUzIDEwLjEyMDUgNjIuNTQxMiAxMC4yMDY0IDYyLjYzMjNDMTAuMjkyNCA2Mi43MjM1IDEwLjM5MTMgNjIuNzkxMiAxMC41MDMzIDYyLjgzNTRDMTAuNjE3OSA2Mi44NzcxIDEwLjc0NDIgNjIuODk3OSAxMC44ODIyIDYyLjg5NzlDMTEuMDU5MyA2Mi44OTc5IDExLjIxNDIgNjIuODY0MSAxMS4zNDcgNjIuNzk2NEMxMS40Nzk5IDYyLjcyODcgMTEuNTkwNSA2Mi42MjMyIDExLjY3OTEgNjIuNDhDMTEuNzcwMiA2Mi4zMzQxIDExLjgzNzkgNjIuMTQ3OSAxMS44ODIyIDYxLjkyMTRDMTEuOTI2NSA2MS42OTIyIDExLjk0ODYgNjEuNDE4OCAxMS45NDg2IDYxLjEwMTFaIiBmaWxsPSJibGFjayIgZmlsbC1vcGFjaXR5PSIwLjc2Ii8+CjxwYXRoIGQ9Ik04LjMxNjQxIDg5LjcwNjNWOTAuM0g0LjIwNzAzVjg5Ljg3NDNMNi43NTM5MSA4NS45MzI5SDcuMzQzNzVMNi43MTA5NCA4Ny4wNzM1TDUuMDI3MzQgODkuNzA2M0g4LjMxNjQxWk03LjUyMzQ0IDg1LjkzMjlWOTEuNjIwNEg2LjgwMDc4Vjg1LjkzMjlINy41MjM0NFpNMTIuNjc1MiA4OC4zMjM1Vjg5LjE5MDdDMTIuNjc1MiA4OS42NTY4IDEyLjYzMzUgOTAuMDUgMTIuNTUwMiA5MC4zNzA0QzEyLjQ2NjggOTAuNjkwNyAxMi4zNDcgOTAuOTQ4NSAxMi4xOTA4IDkxLjE0MzhDMTIuMDM0NSA5MS4zMzkxIDExLjg0NTcgOTEuNDgxIDExLjYyNDQgOTEuNTY5NkMxMS40MDU2IDkxLjY1NTUgMTEuMTU4MiA5MS42OTg1IDEwLjg4MjIgOTEuNjk4NUMxMC42NjM1IDkxLjY5ODUgMTAuNDYxNiA5MS42NzExIDEwLjI3NjcgOTEuNjE2NUMxMC4wOTE4IDkxLjU2MTggOS45MjUxNyA5MS40NzQ1IDkuNzc2NzMgOTEuMzU0N0M5LjYzMDkgOTEuMjMyMyA5LjUwNTkgOTEuMDczNSA5LjQwMTczIDkwLjg3ODJDOS4yOTc1NyA5MC42ODI5IDkuMjE4MTQgOTAuNDQ1OSA5LjE2MzQ1IDkwLjE2NzJDOS4xMDg3NyA4OS44ODg2IDkuMDgxNDIgODkuNTYzMSA5LjA4MTQyIDg5LjE5MDdWODguMzIzNUM5LjA4MTQyIDg3Ljg1NzMgOS4xMjMwOSA4Ny40NjY3IDkuMjA2NDIgODcuMTUxNkM5LjI5MjM2IDg2LjgzNjUgOS40MTM0NSA4Ni41ODM5IDkuNTY5NyA4Ni4zOTM4QzkuNzI1OTUgODYuMjAxMSA5LjkxMzQ1IDg2LjA2MzEgMTAuMTMyMiA4NS45Nzk3QzEwLjM1MzYgODUuODk2NCAxMC42MDEgODUuODU0NyAxMC44NzQ0IDg1Ljg1NDdDMTEuMDk1NyA4NS44NTQ3IDExLjI5ODkgODUuODgyMSAxMS40ODM4IDg1LjkzNjhDMTEuNjcxMyA4NS45ODg5IDExLjgzNzkgODYuMDczNSAxMS45ODM4IDg2LjE5MDdDMTIuMTI5NiA4Ni4zMDUzIDEyLjI1MzMgODYuNDU4OSAxMi4zNTQ5IDg2LjY1MTZDMTIuNDU5IDg2Ljg0MTcgMTIuNTM4NSA4Ny4wNzQ4IDEyLjU5MzEgODcuMzUwOEMxMi42NDc4IDg3LjYyNjkgMTIuNjc1MiA4Ny45NTExIDEyLjY3NTIgODguMzIzNVpNMTEuOTQ4NiA4OS4zMDc5Vjg4LjIwMjRDMTEuOTQ4NiA4Ny45NDcyIDExLjkzMyA4Ny43MjMyIDExLjkwMTcgODcuNTMwNUMxMS44NzMxIDg3LjMzNTIgMTEuODMwMSA4Ny4xNjg1IDExLjc3MjggODcuMDMwNUMxMS43MTU1IDg2Ljg5MjUgMTEuNjQyNiA4Ni43ODA1IDExLjU1NDEgODYuNjk0NkMxMS40NjgxIDg2LjYwODYgMTEuMzY3OSA4Ni41NDYxIDExLjI1MzMgODYuNTA3MUMxMS4xNDEzIDg2LjQ2NTQgMTEuMDE1IDg2LjQ0NDYgMTAuODc0NCA4Ni40NDQ2QzEwLjcwMjUgODYuNDQ0NiAxMC41NTAyIDg2LjQ3NzEgMTAuNDE3NCA4Ni41NDIyQzEwLjI4NDUgODYuNjA0NyAxMC4xNzI2IDg2LjcwNSAxMC4wODE0IDg2Ljg0M0M5Ljk5Mjg4IDg2Ljk4MSA5LjkyNTE3IDg3LjE2MiA5Ljg3ODMgODcuMzg2QzkuODMxNDIgODcuNjA5OSA5LjgwNzk4IDg3Ljg4MjEgOS44MDc5OCA4OC4yMDI0Vjg5LjMwNzlDOS44MDc5OCA4OS41NjMxIDkuODIyMzEgODkuNzg4MyA5Ljg1MDk1IDg5Ljk4MzZDOS44ODIyIDkwLjE3OSA5LjkyNzc4IDkwLjM0ODIgOS45ODc2NyA5MC40OTE1QzEwLjA0NzYgOTAuNjMyMSAxMC4xMjA1IDkwLjc0OCAxMC4yMDY0IDkwLjgzOTFDMTAuMjkyNCA5MC45MzAzIDEwLjM5MTMgOTAuOTk4IDEwLjUwMzMgOTEuMDQyMkMxMC42MTc5IDkxLjA4MzkgMTAuNzQ0MiA5MS4xMDQ3IDEwLjg4MjIgOTEuMTA0N0MxMS4wNTkzIDkxLjEwNDcgMTEuMjE0MiA5MS4wNzA5IDExLjM0NyA5MS4wMDMyQzExLjQ3OTkgOTAuOTM1NSAxMS41OTA1IDkwLjgzIDExLjY3OTEgOTAuNjg2OEMxMS43NzAyIDkwLjU0MDkgMTEuODM3OSA5MC4zNTQ3IDExLjg4MjIgOTAuMTI4MkMxMS45MjY1IDg5Ljg5OSAxMS45NDg2IDg5LjYyNTYgMTEuOTQ4NiA4OS4zMDc5WiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8cGF0aCBkPSJNOC4xOTkyMiAxMTkuMjMzVjExOS44MjdINC40NzY1NlYxMTkuMzA4TDYuMzM5ODQgMTE3LjIzM0M2LjU2OTAxIDExNi45NzggNi43NDYwOSAxMTYuNzYyIDYuODcxMDkgMTE2LjU4NUM2Ljk5ODcgMTE2LjQwNSA3LjA4NzI0IDExNi4yNDUgNy4xMzY3MiAxMTYuMTA0QzcuMTg4OCAxMTUuOTYxIDcuMjE0ODQgMTE1LjgxNSA3LjIxNDg0IDExNS42NjdDNy4yMTQ4NCAxMTUuNDc5IDcuMTc1NzggMTE1LjMxIDcuMDk3NjYgMTE1LjE1OUM3LjAyMjE0IDExNS4wMDYgNi45MTAxNiAxMTQuODgzIDYuNzYxNzIgMTE0Ljc5MkM2LjYxMzI4IDExNC43MDEgNi40MzM1OSAxMTQuNjU1IDYuMjIyNjYgMTE0LjY1NUM1Ljk3MDA1IDExNC42NTUgNS43NTkxMSAxMTQuNzA1IDUuNTg5ODQgMTE0LjgwNEM1LjQyMzE4IDExNC45IDUuMjk4MTggMTE1LjAzNSA1LjIxNDg0IDExNS4yMUM1LjEzMTUxIDExNS4zODQgNS4wODk4NCAxMTUuNTg1IDUuMDg5ODQgMTE1LjgxMkg0LjM2NzE5QzQuMzY3MTkgMTE1LjQ5MSA0LjQzNzUgMTE1LjE5OCA0LjU3ODEyIDExNC45MzNDNC43MTg3NSAxMTQuNjY3IDQuOTI3MDggMTE0LjQ1NiA1LjIwMzEyIDExNC4zQzUuNDc5MTcgMTE0LjE0MSA1LjgxOTAxIDExNC4wNjIgNi4yMjI2NiAxMTQuMDYyQzYuNTgyMDMgMTE0LjA2MiA2Ljg4OTMyIDExNC4xMjUgNy4xNDQ1MyAxMTQuMjUzQzcuMzk5NzQgMTE0LjM3OCA3LjU5NTA1IDExNC41NTUgNy43MzA0NyAxMTQuNzg0QzcuODY4NDkgMTE1LjAxMSA3LjkzNzUgMTE1LjI3NiA3LjkzNzUgMTE1LjU4MUM3LjkzNzUgMTE1Ljc0OCA3LjkwODg1IDExNS45MTcgNy44NTE1NiAxMTYuMDg5QzcuNzk2ODggMTE2LjI1OCA3LjcyMDA1IDExNi40MjcgNy42MjEwOSAxMTYuNTk3QzcuNTI0NzQgMTE2Ljc2NiA3LjQxMTQ2IDExNi45MzMgNy4yODEyNSAxMTcuMDk3QzcuMTUzNjUgMTE3LjI2MSA3LjAxNjkzIDExNy40MjIgNi44NzEwOSAxMTcuNTgxTDUuMzQ3NjYgMTE5LjIzM0g4LjE5OTIyWk0xMi42NzUyIDExNi41M1YxMTcuMzk3QzEyLjY3NTIgMTE3Ljg2NCAxMi42MzM1IDExOC4yNTcgMTIuNTUwMiAxMTguNTc3QzEyLjQ2NjggMTE4Ljg5NyAxMi4zNDcgMTE5LjE1NSAxMi4xOTA4IDExOS4zNTFDMTIuMDM0NSAxMTkuNTQ2IDExLjg0NTcgMTE5LjY4OCAxMS42MjQ0IDExOS43NzZDMTEuNDA1NiAxMTkuODYyIDExLjE1ODIgMTE5LjkwNSAxMC44ODIyIDExOS45MDVDMTAuNjYzNSAxMTkuOTA1IDEwLjQ2MTYgMTE5Ljg3OCAxMC4yNzY3IDExOS44MjNDMTAuMDkxOCAxMTkuNzY5IDkuOTI1MTcgMTE5LjY4MSA5Ljc3NjczIDExOS41NjJDOS42MzA5IDExOS40MzkgOS41MDU5IDExOS4yOCA5LjQwMTczIDExOS4wODVDOS4yOTc1NyAxMTguODkgOS4yMTgxNCAxMTguNjUzIDkuMTYzNDUgMTE4LjM3NEM5LjEwODc3IDExOC4wOTUgOS4wODE0MiAxMTcuNzcgOS4wODE0MiAxMTcuMzk3VjExNi41M0M5LjA4MTQyIDExNi4wNjQgOS4xMjMwOSAxMTUuNjc0IDkuMjA2NDIgMTE1LjM1OEM5LjI5MjM2IDExNS4wNDMgOS40MTM0NSAxMTQuNzkxIDkuNTY5NyAxMTQuNjAxQzkuNzI1OTUgMTE0LjQwOCA5LjkxMzQ1IDExNC4yNyAxMC4xMzIyIDExNC4xODdDMTAuMzUzNiAxMTQuMTAzIDEwLjYwMSAxMTQuMDYyIDEwLjg3NDQgMTE0LjA2MkMxMS4wOTU3IDExNC4wNjIgMTEuMjk4OSAxMTQuMDg5IDExLjQ4MzggMTE0LjE0NEMxMS42NzEzIDExNC4xOTYgMTEuODM3OSAxMTQuMjggMTEuOTgzOCAxMTQuMzk3QzEyLjEyOTYgMTE0LjUxMiAxMi4yNTMzIDExNC42NjYgMTIuMzU0OSAxMTQuODU4QzEyLjQ1OSAxMTUuMDQ5IDEyLjUzODUgMTE1LjI4MiAxMi41OTMxIDExNS41NThDMTIuNjQ3OCAxMTUuODM0IDEyLjY3NTIgMTE2LjE1OCAxMi42NzUyIDExNi41M1pNMTEuOTQ4NiAxMTcuNTE1VjExNi40MDlDMTEuOTQ4NiAxMTYuMTU0IDExLjkzMyAxMTUuOTMgMTEuOTAxNyAxMTUuNzM3QzExLjg3MzEgMTE1LjU0MiAxMS44MzAxIDExNS4zNzUgMTEuNzcyOCAxMTUuMjM3QzExLjcxNTUgMTE1LjA5OSAxMS42NDI2IDExNC45ODcgMTEuNTU0MSAxMTQuOTAxQzExLjQ2ODEgMTE0LjgxNSAxMS4zNjc5IDExNC43NTMgMTEuMjUzMyAxMTQuNzE0QzExLjE0MTMgMTE0LjY3MiAxMS4wMTUgMTE0LjY1MSAxMC44NzQ0IDExNC42NTFDMTAuNzAyNSAxMTQuNjUxIDEwLjU1MDIgMTE0LjY4NCAxMC40MTc0IDExNC43NDlDMTAuMjg0NSAxMTQuODEyIDEwLjE3MjYgMTE0LjkxMiAxMC4wODE0IDExNS4wNUM5Ljk5Mjg4IDExNS4xODggOS45MjUxNyAxMTUuMzY5IDkuODc4MyAxMTUuNTkzQzkuODMxNDIgMTE1LjgxNyA5LjgwNzk4IDExNi4wODkgOS44MDc5OCAxMTYuNDA5VjExNy41MTVDOS44MDc5OCAxMTcuNzcgOS44MjIzMSAxMTcuOTk1IDkuODUwOTUgMTE4LjE5QzkuODgyMiAxMTguMzg2IDkuOTI3NzggMTE4LjU1NSA5Ljk4NzY3IDExOC42OThDMTAuMDQ3NiAxMTguODM5IDEwLjEyMDUgMTE4Ljk1NSAxMC4yMDY0IDExOS4wNDZDMTAuMjkyNCAxMTkuMTM3IDEwLjM5MTMgMTE5LjIwNSAxMC41MDMzIDExOS4yNDlDMTAuNjE3OSAxMTkuMjkxIDEwLjc0NDIgMTE5LjMxMiAxMC44ODIyIDExOS4zMTJDMTEuMDU5MyAxMTkuMzEyIDExLjIxNDIgMTE5LjI3OCAxMS4zNDcgMTE5LjIxQzExLjQ3OTkgMTE5LjE0MiAxMS41OTA1IDExOS4wMzcgMTEuNjc5MSAxMTguODk0QzExLjc3MDIgMTE4Ljc0OCAxMS44Mzc5IDExOC41NjIgMTEuODgyMiAxMTguMzM1QzExLjkyNjUgMTE4LjEwNiAxMS45NDg2IDExNy44MzIgMTEuOTQ4NiAxMTcuNTE1WiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8cGF0aCBkPSJNMTMuMDQzIDE0NC43MzdWMTQ1LjYwNEMxMy4wNDMgMTQ2LjA3IDEzLjAwMTMgMTQ2LjQ2NCAxMi45MTggMTQ2Ljc4NEMxMi44MzQ2IDE0Ny4xMDQgMTIuNzE0OCAxNDcuMzYyIDEyLjU1ODYgMTQ3LjU1N0MxMi40MDIzIDE0Ny43NTMgMTIuMjEzNSAxNDcuODk1IDExLjk5MjIgMTQ3Ljk4M0MxMS43NzM0IDE0OC4wNjkgMTEuNTI2IDE0OC4xMTIgMTEuMjUgMTQ4LjExMkMxMS4wMzEyIDE0OC4xMTIgMTAuODI5NCAxNDguMDg1IDEwLjY0NDUgMTQ4LjAzQzEwLjQ1OTYgMTQ3Ljk3NSAxMC4yOTMgMTQ3Ljg4OCAxMC4xNDQ1IDE0Ny43NjhDOS45OTg3IDE0Ny42NDYgOS44NzM3IDE0Ny40ODcgOS43Njk1MyAxNDcuMjkyQzkuNjY1MzYgMTQ3LjA5NiA5LjU4NTk0IDE0Ni44NTkgOS41MzEyNSAxNDYuNTgxQzkuNDc2NTYgMTQ2LjMwMiA5LjQ0OTIyIDE0NS45NzcgOS40NDkyMiAxNDUuNjA0VjE0NC43MzdDOS40NDkyMiAxNDQuMjcxIDkuNDkwODkgMTQzLjg4IDkuNTc0MjIgMTQzLjU2NUM5LjY2MDE2IDE0My4yNSA5Ljc4MTI1IDE0Mi45OTcgOS45Mzc1IDE0Mi44MDdDMTAuMDkzOCAxNDIuNjE1IDEwLjI4MTIgMTQyLjQ3NyAxMC41IDE0Mi4zOTNDMTAuNzIxNCAxNDIuMzEgMTAuOTY4OCAxNDIuMjY4IDExLjI0MjIgMTQyLjI2OEMxMS40NjM1IDE0Mi4yNjggMTEuNjY2NyAxNDIuMjk2IDExLjg1MTYgMTQyLjM1QzEyLjAzOTEgMTQyLjQwMiAxMi4yMDU3IDE0Mi40ODcgMTIuMzUxNiAxNDIuNjA0QzEyLjQ5NzQgMTQyLjcxOSAxMi42MjExIDE0Mi44NzIgMTIuNzIyNyAxNDMuMDY1QzEyLjgyNjggMTQzLjI1NSAxMi45MDYyIDE0My40ODggMTIuOTYwOSAxNDMuNzY0QzEzLjAxNTYgMTQ0LjA0IDEzLjA0MyAxNDQuMzY1IDEzLjA0MyAxNDQuNzM3Wk0xMi4zMTY0IDE0NS43MjFWMTQ0LjYxNkMxMi4zMTY0IDE0NC4zNjEgMTIuMzAwOCAxNDQuMTM3IDEyLjI2OTUgMTQzLjk0NEMxMi4yNDA5IDE0My43NDkgMTIuMTk3OSAxNDMuNTgyIDEyLjE0MDYgMTQzLjQ0NEMxMi4wODMzIDE0My4zMDYgMTIuMDEwNCAxNDMuMTk0IDExLjkyMTkgMTQzLjEwOEMxMS44MzU5IDE0My4wMjIgMTEuNzM1NyAxNDIuOTYgMTEuNjIxMSAxNDIuOTIxQzExLjUwOTEgMTQyLjg3OSAxMS4zODI4IDE0Mi44NTggMTEuMjQyMiAxNDIuODU4QzExLjA3MDMgMTQyLjg1OCAxMC45MTggMTQyLjg5MSAxMC43ODUyIDE0Mi45NTZDMTAuNjUyMyAxNDMuMDE4IDEwLjU0MDQgMTQzLjExOSAxMC40NDkyIDE0My4yNTdDMTAuMzYwNyAxNDMuMzk1IDEwLjI5MyAxNDMuNTc2IDEwLjI0NjEgMTQzLjhDMTAuMTk5MiAxNDQuMDI0IDEwLjE3NTggMTQ0LjI5NiAxMC4xNzU4IDE0NC42MTZWMTQ1LjcyMUMxMC4xNzU4IDE0NS45NzcgMTAuMTkwMSAxNDYuMjAyIDEwLjIxODggMTQ2LjM5N0MxMC4yNSAxNDYuNTkzIDEwLjI5NTYgMTQ2Ljc2MiAxMC4zNTU1IDE0Ni45MDVDMTAuNDE1NCAxNDcuMDQ2IDEwLjQ4ODMgMTQ3LjE2MiAxMC41NzQyIDE0Ny4yNTNDMTAuNjYwMiAxNDcuMzQ0IDEwLjc1OTEgMTQ3LjQxMiAxMC44NzExIDE0Ny40NTZDMTAuOTg1NyAxNDcuNDk3IDExLjExMiAxNDcuNTE4IDExLjI1IDE0Ny41MThDMTEuNDI3MSAxNDcuNTE4IDExLjU4MiAxNDcuNDg0IDExLjcxNDggMTQ3LjQxN0MxMS44NDc3IDE0Ny4zNDkgMTEuOTU4MyAxNDcuMjQ0IDEyLjA0NjkgMTQ3LjFDMTIuMTM4IDE0Ni45NTUgMTIuMjA1NyAxNDYuNzY4IDEyLjI1IDE0Ni41NDJDMTIuMjk0MyAxNDYuMzEzIDEyLjMxNjQgMTQ2LjAzOSAxMi4zMTY0IDE0NS43MjFaIiBmaWxsPSJibGFjayIgZmlsbC1vcGFjaXR5PSIwLjc2Ii8+CjxwYXRoIGQ9Ik0xOSA0LjE2MTEzTDE5NCA0LjE2MTE2IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC4xMiIvPgo8cGF0aCBkPSJNMTkgMzMuMTYxMUwxOTQgMzMuMTYxMiIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLW9wYWNpdHk9IjAuMTIiLz4KPHBhdGggZD0iTTE5IDYxLjE2MTFMMTk0IDYxLjE2MTIiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS1vcGFjaXR5PSIwLjEyIi8+CjxwYXRoIGQ9Ik0xOSA4OS4xNjExTDE5NCA4OS4xNjEyIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC4xMiIvPgo8cGF0aCBkPSJNMTkgMTE4LjE2MUwxOTQgMTE4LjE2MSIgc3Ryb2tlPSJibGFjayIgc3Ryb2tlLW9wYWNpdHk9IjAuMTIiLz4KPHBhdGggZD0iTTI4IDI3QzI4IDI1Ljg5NTQgMjguODk1NCAyNSAzMCAyNUg1NkM1Ny4xMDQ2IDI1IDU4IDI1Ljg5NTQgNTggMjdWMTQ2SDI4VjI3WiIgZmlsbD0iIzA4ODcyQiIvPgo8cGF0aCBkPSJNNzAgNjlDNzAgNjcuODk1NCA3MC44OTU0IDY3IDcyIDY3SDk4Qzk5LjEwNDYgNjcgMTAwIDY3Ljg5NTQgMTAwIDY5VjE0Nkg3MFY2OVoiIGZpbGw9IiM0QjkzRkYiLz4KPHBhdGggZD0iTTExMiA5MkMxMTIgOTAuODk1NCAxMTIuODk1IDkwIDExNCA5MEgxNDBDMTQxLjEwNSA5MCAxNDIgOTAuODk1NCAxNDIgOTJWMTQ2SDExMlY5MloiIGZpbGw9IiNGRjRENUEiLz4KPHBhdGggZD0iTTE1NCA3OUMxNTQgNzcuODk1NCAxNTQuODk1IDc3IDE1NiA3N0gxODJDMTgzLjEwNSA3NyAxODQgNzcuODk1NCAxODQgNzlWMTQ2SDE1NFY3OVoiIGZpbGw9IiNGRkMxMDciLz4KPGxpbmUgeDE9IjE3LjIiIHkxPSIxNDUuOTYxIiB4Mj0iMTk2LjgiIHkyPSIxNDUuOTYxIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC43IiBzdHJva2Utd2lkdGg9IjAuNCIgc3Ryb2tlLWxpbmVjYXA9InNxdWFyZSIvPgo8cGF0aCBkPSJNNDMuMTMyOCAxNC4xNzU4TDQxLjUxMTcgMTlINDAuMzc1TDQyLjUgMTMuMzEyNUg0My4yMjY2TDQzLjEzMjggMTQuMTc1OFpNNDQuNDg4MyAxOUw0Mi44NTk0IDE0LjE3NThMNDIuNzYxNyAxMy4zMTI1SDQzLjQ5MjJMNDUuNjI4OSAxOUg0NC40ODgzWk00NC40MTQxIDE2Ljg4NjdWMTcuNzM0NEg0MS4zNjMzVjE2Ljg4NjdINDQuNDE0MVoiIGZpbGw9ImJsYWNrIiBmaWxsLW9wYWNpdHk9IjAuNzYiLz4KPHBhdGggZD0iTTg1LjE4MzYgNTguNDY0OEg4My43MzA1TDgzLjcyMjcgNTcuNzA3SDg0Ljk2ODhDODUuMTgyMyA1Ny43MDcgODUuMzYwNyA1Ny42NzcxIDg1LjUwMzkgNTcuNjE3MkM4NS42NDcxIDU3LjU1NzMgODUuNzU1MiA1Ny40NzAxIDg1LjgyODEgNTcuMzU1NUM4NS45MDM2IDU3LjI0MDkgODUuOTQxNCA1Ny4xMDE2IDg1Ljk0MTQgNTYuOTM3NUM4NS45NDE0IDU2Ljc1NTIgODUuOTA2MiA1Ni42MDY4IDg1LjgzNTkgNTYuNDkyMkM4NS43NjgyIDU2LjM3NzYgODUuNjYxNSA1Ni4yOTQzIDg1LjUxNTYgNTYuMjQyMkM4NS4zNjk4IDU2LjE4NzUgODUuMTg0OSA1Ni4xNjAyIDg0Ljk2MDkgNTYuMTYwMkg4NC4wNzgxVjYxSDgzLjAwMzlWNTUuMzEyNUg4NC45NjA5Qzg1LjI4MzkgNTUuMzEyNSA4NS41NzE2IDU1LjM0MzggODUuODI0MiA1NS40MDYyQzg2LjA3OTQgNTUuNDY2MSA4Ni4yOTU2IDU1LjU1OTkgODYuNDcyNyA1NS42ODc1Qzg2LjY0OTcgNTUuODE1MSA4Ni43ODM5IDU1Ljk3NTMgODYuODc1IDU2LjE2OEM4Ni45Njg4IDU2LjM2MDcgODcuMDE1NiA1Ni41ODk4IDg3LjAxNTYgNTYuODU1NUM4Ny4wMTU2IDU3LjA4OTggODYuOTYwOSA1Ny4zMDYgODYuODUxNiA1Ny41MDM5Qzg2Ljc0NDggNTcuNjk5MiA4Ni41Nzk0IDU3Ljg1ODEgODYuMzU1NSA1Ny45ODA1Qzg2LjEzNDEgNTguMTAyOSA4NS44NTI5IDU4LjE3MzIgODUuNTExNyA1OC4xOTE0TDg1LjE4MzYgNTguNDY0OFpNODUuMTM2NyA2MUg4My40MTQxTDgzLjg2MzMgNjAuMTU2Mkg4NS4xMzY3Qzg1LjM1MDMgNjAuMTU2MiA4NS41MjYgNjAuMTIxMSA4NS42NjQxIDYwLjA1MDhDODUuODA0NyA1OS45ODA1IDg1LjkwODkgNTkuODg0MSA4NS45NzY2IDU5Ljc2MTdDODYuMDQ2OSA1OS42MzY3IDg2LjA4MiA1OS40OTM1IDg2LjA4MiA1OS4zMzJDODYuMDgyIDU5LjE1NDkgODYuMDUwOCA1OS4wMDEzIDg1Ljk4ODMgNTguODcxMUM4NS45Mjg0IDU4Ljc0MDkgODUuODMyIDU4LjY0MDYgODUuNjk5MiA1OC41NzAzQzg1LjU2OSA1OC41IDg1LjM5NzEgNTguNDY0OCA4NS4xODM2IDU4LjQ2NDhIODQuMDY2NEw4NC4wNzQyIDU3LjcwN0g4NS40OTYxTDg1Ljc0MjIgNThDODYuMDcwMyA1OC4wMDI2IDg2LjMzNzIgNTguMDY3NyA4Ni41NDMgNTguMTk1M0M4Ni43NTEzIDU4LjMyMjkgODYuOTA0OSA1OC40ODcgODcuMDAzOSA1OC42ODc1Qzg3LjEwMjkgNTguODg4IDg3LjE1MjMgNTkuMTA0MiA4Ny4xNTIzIDU5LjMzNTlDODcuMTUyMyA1OS43MDA1IDg3LjA3MjkgNjAuMDA2NSA4Ni45MTQxIDYwLjI1MzlDODYuNzU3OCA2MC41MDEzIDg2LjUyODYgNjAuNjg3NSA4Ni4yMjY2IDYwLjgxMjVDODUuOTI3MSA2MC45Mzc1IDg1LjU2MzggNjEgODUuMTM2NyA2MVoiIGZpbGw9ImJsYWNrIiBmaWxsLW9wYWNpdHk9IjAuNzYiLz4KPHBhdGggZD0iTTE2OC42MzcgNzJIMTY3LjQxTDE2Ny40MTggNzEuMTU2MkgxNjguNjM3QzE2OC45NjcgNzEuMTU2MiAxNjkuMjQ1IDcxLjA4MzMgMTY5LjQ2OSA3MC45Mzc1QzE2OS42OTMgNzAuNzg5MSAxNjkuODYyIDcwLjU3NjggMTY5Ljk3NyA3MC4zMDA4QzE3MC4wOTEgNzAuMDIyMSAxNzAuMTQ4IDY5LjY4ODggMTcwLjE0OCA2OS4zMDA4VjY5LjAwNzhDMTcwLjE0OCA2OC43MDgzIDE3MC4xMTYgNjguNDQ0IDE3MC4wNTEgNjguMjE0OEMxNjkuOTg2IDY3Ljk4NTcgMTY5Ljg4OSA2Ny43OTMgMTY5Ljc2MiA2Ny42MzY3QzE2OS42MzcgNjcuNDgwNSAxNjkuNDgyIDY3LjM2MiAxNjkuMjk3IDY3LjI4MTJDMTY5LjExMiA2Ny4yMDA1IDE2OC45IDY3LjE2MDIgMTY4LjY2IDY3LjE2MDJIMTY3LjM4N1Y2Ni4zMTI1SDE2OC42NkMxNjkuMDQgNjYuMzEyNSAxNjkuMzg3IDY2LjM3NjMgMTY5LjY5OSA2Ni41MDM5QzE3MC4wMTQgNjYuNjMxNSAxNzAuMjg2IDY2LjgxNTEgMTcwLjUxNiA2Ny4wNTQ3QzE3MC43NDcgNjcuMjkxNyAxNzAuOTI0IDY3LjU3NTUgMTcxLjA0NyA2Ny45MDYyQzE3MS4xNzIgNjguMjM3IDE3MS4yMzQgNjguNjA2OCAxNzEuMjM0IDY5LjAxNTZWNjkuMzAwOEMxNzEuMjM0IDY5LjcwNyAxNzEuMTcyIDcwLjA3NjggMTcxLjA0NyA3MC40MTAyQzE3MC45MjQgNzAuNzQwOSAxNzAuNzQ3IDcxLjAyNDcgMTcwLjUxNiA3MS4yNjE3QzE3MC4yODYgNzEuNDk4NyAxNzAuMDEzIDcxLjY4MSAxNjkuNjk1IDcxLjgwODZDMTY5LjM3OCA3MS45MzYyIDE2OS4wMjUgNzIgMTY4LjYzNyA3MlpNMTY4LjAxMiA2Ni4zMTI1VjcySDE2Ni45MzhWNjYuMzEyNUgxNjguMDEyWiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8cGF0aCBkPSJNMTI4LjI1NCA4Mi4xMjg5SDEyOS4zMjhDMTI5LjI5OSA4Mi41MDkxIDEyOS4xOTQgODIuODQ2NCAxMjkuMDEyIDgzLjE0MDZDMTI4LjgyOSA4My40MzIzIDEyOC41NzYgODMuNjYxNSAxMjguMjUgODMuODI4MUMxMjcuOTI0IDgzLjk5NDggMTI3LjUzIDg0LjA3ODEgMTI3LjA2NiA4NC4wNzgxQzEyNi43MSA4NC4wNzgxIDEyNi4zODggODQuMDE1NiAxMjYuMTAyIDgzLjg5MDZDMTI1LjgxOCA4My43NjMgMTI1LjU3NCA4My41ODIgMTI1LjM3MSA4My4zNDc3QzEyNS4xNzEgODMuMTEwNyAxMjUuMDE3IDgyLjgyNjggMTI0LjkxIDgyLjQ5NjFDMTI0LjgwMyA4Mi4xNjI4IDEyNC43NSA4MS43ODkxIDEyNC43NSA4MS4zNzVWODAuOTQxNEMxMjQuNzUgODAuNTI3MyAxMjQuODA1IDgwLjE1MzYgMTI0LjkxNCA3OS44MjAzQzEyNS4wMjMgNzkuNDg3IDEyNS4xOCA3OS4yMDMxIDEyNS4zODMgNzguOTY4OEMxMjUuNTg5IDc4LjczMTggMTI1LjgzNSA3OC41NDk1IDEyNi4xMjEgNzguNDIxOUMxMjYuNDEgNzguMjk0MyAxMjYuNzMzIDc4LjIzMDUgMTI3LjA5IDc4LjIzMDVDMTI3LjU1MyA3OC4yMzA1IDEyNy45NDUgNzguMzE2NCAxMjguMjY2IDc4LjQ4ODNDMTI4LjU4NiA3OC42NTc2IDEyOC44MzUgNzguODkwNiAxMjkuMDEyIDc5LjE4NzVDMTI5LjE4OSA3OS40ODQ0IDEyOS4yOTYgNzkuODI1NSAxMjkuMzMyIDgwLjIxMDlIMTI4LjI1OEMxMjguMjM3IDc5Ljk3MTQgMTI4LjE4NSA3OS43NjgyIDEyOC4xMDIgNzkuNjAxNkMxMjguMDIxIDc5LjQzNDkgMTI3Ljg5OCA3OS4zMDg2IDEyNy43MzQgNzkuMjIyN0MxMjcuNTczIDc5LjEzNDEgMTI3LjM1OCA3OS4wODk4IDEyNy4wOSA3OS4wODk4QzEyNi44ODIgNzkuMDg5OCAxMjYuNjk4IDc5LjEyODkgMTI2LjUzOSA3OS4yMDdDMTI2LjM4MyA3OS4yODUyIDEyNi4yNTMgNzkuNDAyMyAxMjYuMTQ4IDc5LjU1ODZDMTI2LjA0NCA3OS43MTIyIDEyNS45NjYgNzkuOTA0OSAxMjUuOTE0IDgwLjEzNjdDMTI1Ljg2MiA4MC4zNjU5IDEyNS44MzYgODAuNjMxNSAxMjUuODM2IDgwLjkzMzZWODEuMzc1QzEyNS44MzYgODEuNjY0MSAxMjUuODU5IDgxLjkyMzIgMTI1LjkwNiA4Mi4xNTIzQzEyNS45NTMgODIuMzgxNSAxMjYuMDI2IDgyLjU3NTUgMTI2LjEyNSA4Mi43MzQ0QzEyNi4yMjQgODIuODkzMiAxMjYuMzUyIDgzLjAxNDMgMTI2LjUwOCA4My4wOTc3QzEyNi42NjQgODMuMTgxIDEyNi44NSA4My4yMjI3IDEyNy4wNjYgODMuMjIyN0MxMjcuMzI5IDgzLjIyMjcgMTI3LjU0MyA4My4xODEgMTI3LjcwNyA4My4wOTc3QzEyNy44NzQgODMuMDE0MyAxMjggODIuODkxOSAxMjguMDg2IDgyLjczMDVDMTI4LjE3NCA4Mi41NjkgMTI4LjIzIDgyLjM2ODUgMTI4LjI1NCA4Mi4xMjg5WiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8L2c+CjxkZWZzPgo8Y2xpcFBhdGggaWQ9ImNsaXAwXzQ2MjRfMzgwMjIiPgo8cmVjdCB3aWR0aD0iMjAwIiBoZWlnaHQ9IjE2MCIgZmlsbD0id2hpdGUiLz4KPC9jbGlwUGF0aD4KPC9kZWZzPgo8L3N2Zz4K", + "description": "Displays the latest values of the attributes or time-series data in a bar chart. Supports numeric values only.", "descriptor": { "type": "latest", - "sizeX": 7, - "sizeY": 5, - "resources": [ - { - "url": "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.3.0/Chart.min.js" - } - ], - "templateHtml": "\n", + "sizeX": 5, + "sizeY": 4, + "resources": [], + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n $scope = self.ctx.$scope;\n utils = $scope.$injector.get(self.ctx.servicesMap.get('utils'));\n settings = utils.deepClone(self.ctx.settings) || {};\n settings.showTooltip = utils.defaultValue(settings.showTooltip, true);\n \n Chart.defaults.global.tooltips.enabled = settings.showTooltip;\n \n var barData = {\n labels: [],\n datasets: []\n };\n \n for (var i = 0; i < self.ctx.datasources.length; i++) {\n var datasource = self.ctx.datasources[i];\n for (var d = 0; d < datasource.dataKeys.length; d++) {\n var dataKey = datasource.dataKeys[d];\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n var dataset = {\n label: dataKey.label + units,\n data: [0],\n backgroundColor: [dataKey.color],\n borderColor: [dataKey.color],\n borderWidth: 1\n }\n barData.datasets.push(dataset);\n }\n }\n\n var ctx = $('#barChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'bar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scales: {\n yAxes: [{\n ticks: {\n beginAtZero:true\n }\n }]\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n var c = 0;\n for (var i = 0; i < self.ctx.chart.data.datasets.length; i++) {\n var dataset = self.ctx.chart.data.datasets[i];\n var cellData = self.ctx.data[i]; \n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n dataset.data[0] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\nself.onDestroy = function() {\n self.ctx.chart.destroy();\n self.ctx.chart = null;\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.barChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.barChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", "settingsSchema": "", - "dataKeySettingsSchema": "{}\n", - "settingsDirective": "tb-chart-widget-settings", - "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Bars\"}" + "dataKeySettingsSchema": "", + "settingsDirective": "tb-bar-chart-widget-settings", + "hasBasicMode": true, + "basicModeDirective": "tb-bar-chart-basic-config", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Wind\",\"color\":\"#08872B\",\"settings\":{},\"_hash\":0.7227918773301678,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Solar\",\"color\":\"#FF4D5A\",\"settings\":{},\"_hash\":0.2392660816082064,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Hydroelectric\",\"color\":\"#FFDE30\",\"settings\":{},\"_hash\":0.7051898468567794,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"decimals\":0,\"aggregationType\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":null,\"padding\":\"0px\",\"settings\":{},\"title\":\"Bars\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"units\":null,\"decimals\":0,\"useDashboardTimewindow\":true,\"showLegend\":false,\"widgetStyle\":{},\"actions\":{\"headerButton\":[]},\"configMode\":\"basic\",\"displayTimewindow\":true,\"margin\":\"0px\",\"borderRadius\":\"\",\"widgetCss\":\"\",\"pageSize\":1024,\"noDataDisplayMessage\":\"\",\"showTitleIcon\":false,\"titleTooltip\":\"\",\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleIcon\":\"bar_chart\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true}}" }, - "externalId": null, "tags": [ + "bars", "bar", "bar chart" ] diff --git a/application/src/main/data/json/system/widget_types/bars_deprecated.json b/application/src/main/data/json/system/widget_types/bars_deprecated.json new file mode 100644 index 0000000000..6324f9d7b2 --- /dev/null +++ b/application/src/main/data/json/system/widget_types/bars_deprecated.json @@ -0,0 +1,29 @@ +{ + "fqn": "charts.bars", + "name": "Bars", + "deprecated": true, + "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAMAAAB+IdObAAAA8FBMVEUhlvNMr1Bqamp5eXl7e3t8fHx9fX1+fn5/f3+AgICCgoKDg4OEhISGhoaHh4eKioqMjIyNjY2Ojo6QkJCRkZGSkpKWlpaXl5ebm5udnZ2enp6goKChoaGkpKSnp6epqamsrKyurq6xsbGzs7O1tbW2tra3t7e4uLi7u7u9vb3BwcHCwsLDw8PGxsbKysrNzc3Ozs7R0dHS0tLT09PZ2dna2trc3Nzd3d3e3t7g4ODh4eHj4+Pk5OTm5ubn5+fo6Ojp6enu7u7w8PDz8/P0Qzb09PT29vb39/f5+fn6+vr7+/v8/Pz9/f3+/v7/wQf///+dc+aLAAAAAWJLR0RPbmZBSQAAAcFJREFUeNrt3ds2AgEYhuHsaSOZbAvZi0r2YYjCJOW7/7txZhkcDNbM6h/vdwfPmlX/ybtmEorJErGCeJLadz3rkKPpZamaLTp925DHdFvSpKelU9uQ/cLKhtcdk7YqtiHruevtojch7ZZtQ0o1dcdfRqXNqm1IbVU3OaVamm/YhvQW5zIXOknnC5JUt7qEpE5fUv/5HVePy2UHAgQIECBAgAABAgQIECBxgrwGHBAgQIAAAQIECBAgQIAAAQIECJC/QRIBN0iQ+66voDMLuRp2fQWdVUhvNun6CjqrkJ0Dx/UVdEYhzXzfcX0FnVFIrlSZ2mx/LOiMQuqHh6k972NBZ/fv13G/KeiCQkIu4358EL8UdBafSGwuOxAgQIAAAQIECJDYQB4CDggQIECAAAECBAgQIECAAAECBAgQIECA/BrSufn0DjqjkEZmLXkWaUEXEmThXMeFSAu68H4j5b1IC7rQILfZTqQFXViQ1nRTL1EWdCFBnmYuJUVZ0IUEWR1xHKcXZUEXFDLwBR2XHQgQIECAAAEC5H9ChgIOCBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgxiBmv+L6Bl9pkxYph15gAAAAAElFTkSuQmCC", + "description": "Displays latest values of the attributes or time-series data for multiple entities as separate bars.", + "descriptor": { + "type": "latest", + "sizeX": 7, + "sizeY": 5, + "resources": [ + { + "url": "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.3.0/Chart.min.js" + } + ], + "templateHtml": "\n", + "templateCss": "", + "controllerScript": "self.onInit = function() {\n $scope = self.ctx.$scope;\n utils = $scope.$injector.get(self.ctx.servicesMap.get('utils'));\n settings = utils.deepClone(self.ctx.settings) || {};\n settings.showTooltip = utils.defaultValue(settings.showTooltip, true);\n \n Chart.defaults.global.tooltips.enabled = settings.showTooltip;\n \n var barData = {\n labels: [],\n datasets: []\n };\n \n for (var i = 0; i < self.ctx.datasources.length; i++) {\n var datasource = self.ctx.datasources[i];\n for (var d = 0; d < datasource.dataKeys.length; d++) {\n var dataKey = datasource.dataKeys[d];\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n var dataset = {\n label: dataKey.label + units,\n data: [0],\n backgroundColor: [dataKey.color],\n borderColor: [dataKey.color],\n borderWidth: 1\n }\n barData.datasets.push(dataset);\n }\n }\n\n var ctx = $('#barChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'bar',\n data: barData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scales: {\n yAxes: [{\n ticks: {\n beginAtZero:true\n }\n }]\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n var c = 0;\n for (var i = 0; i < self.ctx.chart.data.datasets.length; i++) {\n var dataset = self.ctx.chart.data.datasets[i];\n var cellData = self.ctx.data[i]; \n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n dataset.data[0] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n self.ctx.chart.resize();\n}\n\nself.onDestroy = function() {\n self.ctx.chart.destroy();\n self.ctx.chart = null;\n}\n", + "settingsSchema": "", + "dataKeySettingsSchema": "{}\n", + "settingsDirective": "tb-chart-widget-settings", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Bars\"}" + }, + "externalId": null, + "tags": [ + "bar", + "bar chart" + ] +} \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_types/polar_area.json b/application/src/main/data/json/system/widget_types/polar_area.json index 4cc516321c..8d4a0b24a6 100644 --- a/application/src/main/data/json/system/widget_types/polar_area.json +++ b/application/src/main/data/json/system/widget_types/polar_area.json @@ -1,25 +1,28 @@ { - "fqn": "charts.polar_area_chart_js", - "name": "Polar Area", + "fqn": "polar_area", + "name": "Polar area", "deprecated": false, - "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAIAAADGnbT+AAAABmJLR0QA/wD/AP+gvaeTAAAYvUlEQVR42u2deXAU15nA+W+zm73/yV7Zqq09aje7SeXY2kocZ7cSH7GdxCG+cJxybHw7Xt/4WB+AY8CAMeEGc4MNAgcMxhgB5r4RkpAAHYCEDoRu0DX31fuTejyaEWjm9fTr7tejefWKksRMT/f3fvO+733He6O0fMs3C9qovAgytlgs5vf7Ez+3tbX19fXlxZIHK/sWjUYLCgqmTZtWVFSkUzVr1qxVq1ZNnz79yJEjefnkwcqy7d+/f8OGDfCk/1pdXb18+XJ+8Pl8EydOzMsnD1aWberUqZ9//jlTVElJCb8ePHhw27Zt+n+98cYbkUgkL6I8WNm0cePGbd++vbm5+Z133jl//vzu3bt37Nih/9f48eMThle+5cEa1jwPBAI9PT0dHR2XLl1K/P2VV17Rf9i3b19hYeGhQ4e2bt2amLGwwPSfeQtv7O3t5SIJvZkHa4SS5PF4WN/V19djOZWVlVVUVDAnNTQ0tLS0JF6Gqc5f+AFL6+jRoxcuXFiwYAG/guCkSZMSL+MtvIy3cxEuxQX5lYvzESOWs1EjDSb02rlz5xj+qqqqxsbGzs5Or9ebmHuGtKampilTpsydO3fevHnhcJi/rFy5cvbs2ZMnTz59+vRwC0k+hctycT6CD+Lj+NCRBlnug8VIX7lypa6urry8nJFGbaGzDNndKLjkX0OhkDgifBDTGx/KR3MD3AY3MxzHebDcMT91dXWhvBjOmpoaphCAcPaWuAFug5vRCeP2cngOy0GwmGCYIU6dOoUOYiAVdAqgVbkxbg99yq0OmRHzYCnX0HHMByCFbeSK0eImuVVumNvm5vNgKaf1Ll++jBFTWVnJTOA6/cINc9vcPI+ABZYHS4nGSDAkqBVsZLc/C49w9uxZHgfzKw+Wk4qPrzgjkWNKBLzwhPFors6hcCVYmCa1tbV4I3Pgmz1c49HOnDnDqjYYDObBssMcwc2Ntdva2prz/kbcXTwmD8sju+5h3QQWzmvsDxZQVnyJGbjO7p6K2oY9RWUFhXsXrN8yfeVHExeufmnm4t9MnpP8ytf3BaYdDS4qDa6rDBXWhsvaIt0BC0cd7xfzFg/O4+fBkj9RsSzH6yNX93X19p2oOAtGk5aseey3v3vwzXeH68nv+ocFfVf3by/33LnR99KewMpToVPt0Yhs1zprFN3p5ZapywVgkZ2CMcu3Vo/WmWzhSOT0+boPtu5iKkpDklGwhvSvLe4bs9k343iwpDUSlUQCj49liVFPmmEeLLMNBw9GBv+aHpXI8dPV89dteXLSbHGesgYruf/nCs/LewI7LoT9Er4XGvk5CAS/XR6s7NUfCQLYFibz6S61d360c//TU+dnwZMUsBL960s9r+0LFLeYDTEhEFbEFy9eVFktKgoWFisOKtRf1pE+hM4U9daiD8zwJBesRL9tvXf16ZDPxASGWBAOIpJiHowUsPhG4sJhpZ21FXWg5NSrs5ZKQcoKsPT+nRWeecXBHhMrStwQCErNDGnlwMKNjg2RXcgsGo3tKy5/YcYiiUhZB1ZCP04/FuzKFi+WyYhLwXCWWmDpYsoulFF5oWH8/FXSkbIaLL1/c5lnYWkwlJWTAnEhNNWCEAqBpbtqslhLt1/pnr1mk0VI2QOW3m8q8O5tyMamRBsiOqWWiqqAhUMhC3OBVdHeE+WPvz3LUqpsA0vvD33mb/XEsmALAZr3y+QUWHzVEIrR1LzOrp7pK9ZbjZT9YOma8eOzhpd7CBAxKpLR5TxYGAdM40bnqiNlFdm5Ol0Blt6f3uk3GoWELYSpAlsOg6WvAQ3ZVcT8cXjahpSDYNG/v9pT3mbMpEeYWS+AcgQsZimjIrjS00fA2GaqHASL/q/v931UZay+SF8nOhtSdAwsfOsYBIYWMrUXm5+ZNt9+qpwFS+8TDgQiRrQi2hDxOuiXdwYs4i2EI5KL2TM2UhJsWP0pCxb98UJ/wIgvAvEiZKfiic6ARXSZDBDx1xdXnHvkrZlOUaUIWHRScXqDBkAhnkiseqSApZc6iUeXdx0rHTt+hoNUqQOWHsBu9xoo8CcPwhHHqd1g6Qa7uHPh0MkzY8e/6yxVSoFFv2Gtt8MXMyRw+8t3bQULfU8uqLh3mLzhhybMcJwq1cCi/+z3XnGdSG4gYrfZ2LIVLPLW0fri1vojE2eqQJWCYNFJsfeGRFnBok3eUC6nwKLIBKew4AL4QlNL+uqGPFj0sVt9gj4IxI7w2Qks18BiHhavHO/u8zz/7iJ1qFIWLPo7R0Qr4fBsUYhhm0K0CSy2tBP0L1D1MGnxGqWoUhks+ibhcDVDkHVeropgsSRhYSK479myTdtVo0pxsIj5UDQrIlsKfRkIe2r27QBL/ItypLxSQaoUB4v+vdUewTwI3PHi6yelwSJ/AR+diGq/0tP71JS5ebCy6y/uFnINkhtCDNGG3AfLwcJgFLHZIW/q8nVqUuUKsOjbaoSMLYYDt5a7wWIlIvgMhYeKlKXKLWCxf4RgTjOD0t3d7VawdBeDyK5ol7t7HcxcyBmw6E/tEFKIlIsxNG4Fi9gnOziKvHLeuk9UpspFYNH3NwqtEMmosTSD2UKwsK5ECikraxsUp8pdYN1c4A1HhSYtSy0tq8BCA4pMtpFI9LU5y/Ngye1Ly4RchgyQdctDq8Bi3z2RNCAq4tWnynVgfWOpR2RLCP2YDDeBpbvaM/qumK4M7X6WB0u8zzqR2b2OT8u6VC1LwLo00DK+jD1hXEGVG8ESnLRIZLIonUY+WExUZGhk/B6wM4zcnYbyYA3pc4uDIrqFwbIi5UE+WDh2RbwMR1UNC+YMWN9a5hHJBGSwrHCWygeLGKdI8vHkpWvzYFndRSpdSVzmjDvVwcIe5DC+jBU4zR2XVSiRyHmwRm/IXAxNcilDJv1sTslg4cwVWcGu2bbbRVS5Fyz66fbMxHCatfR92ySDxaSaUQ+GwmFl02NyDyzO0XBEG8oEi8WFSKZofxFY3UUmLSs2C82DlejXrfb89lCgtDVz6JAhY+DUBYs6HOKDxjxe7Z2b9xx+dfayPFiy+vUf9PPEbvKGXAiEd+TW8MgEi4qJrL1t9c1tG3cdVDZuqD5Y7F/63vFgRUeWNjieUkN7tNgKFh4R82dSslPtzqMlOCOUWjYqC9bN67xEb0Qs9IzJDpjwKoKF5VRWVpZ5q4+eI1rpD7SOzVooQ4i6o0shwlQDS+eptkuaj4CBw+kg0QUvDSxRA6thirb/D+K96D+05qVaqC39O8gvhTBOD3xownsjGax/XNi/ZQM81XfLPrTui/w5iWaWNLDa2trY9Srz607dOgjWIGH/rrWu1ELt6d/a5/UdLquY9eHHD098b+SA9U8L++7e5FtxKpTFHt2GGsPX3t6uHFj19fWZIzmxoHboL68BVqIf/xetabbmz1D45vH5dcJs2zXEfrASPInvhmWy4c1qaGhQDiyhibT7cDqqkvuRv9MaZ2oBnjOdWL3+QNGZ6iUbt1ldi2EbWJQ1P7LNzybvIkkv7NYnsawZY0ZisrIcsHTLPXO8qWm+KFiJfvgr/WZZ/xyWTtDBULisugbCLNr83Wqw/m1xnKc+IztBisQ5DNnvDKIs+10OWKT1UO4s4JB4yjBYiX7wT7Tzz2h95Vosmj5eBGErP9lh8uRLe8C6/gMv+ekNPbFQJJvhhCq5oZgszgexFixRL0jZD7MHa5CwP9Vqx2meM+kJI5HwXEMTgaNnpy1QDawfrvGurwp1+szODehB0vRU80TKBEvU7jv81xLAGuxf7p/Deo5rsZAIYc9NX+gsWD9a6y2oCHX4ZBrjOJ8kbubOIMrSrXLAIpKTOSAQbJFKVXL/klZ5f7/rNS1hWA91l1oJTb78uyV2gjV6o3f7hXBXwJLFHXWnGN2yrsYgEpdTCCx8DZmLvXpPWAZWUj9zl9a9X4sG5Aa/swDr3s2+XfVhT8haZ4GQl8eI0cYFFQKL5L7MRc+dW+0AK9HLbuoPHEV6RQibuHC1LLAe+NS/tyHsC2fg6UJ33YeVa2cUzzQpeTYek1hmQ/K7oYMdLAcL/0dmJ1bLclvBSvTym/qZjmQo+U0f/M4I1qOF/mOXIsFMkdLG3sa1Veue2PXUTzeNpt+5ZYzJ5T2Zn7JQ0F1Z6FaFwMLXkHmZWj/JGbASvfR6keA352teTdhwYOk8pT/LGXTOd9XA02OfP6nzlNybPc2KoGDAbWQbWOQfZl6b1LzgMFiJfuJbWusHWiiDacIOg5wLTOCI4Hfy3/95Ud+bB/ozM9PvvRGNRSs7qxafWvpA4cNX85ToR5uPmZE8Z8cZTa5M5wUMhWT5L+SAJVKZo537X1XASg5+t2QOfrM9eCox6V4cjoaLW0vnlM6/77Nfp+Ep0bfUbDXpysKrKdH5zlAqBJZQPOfsE8qBlRz8vrRAC2R/UFYwEjrWfPy94lljtv5KhKdE33DuYzOSR1FITFdnEBlKhcA6efJkZiO0+mF1wRoMTf6VSGgyiadgUcuJmcWz7/n0PkM8JXpB1XozkkfsCF8WWBKvZiNYVQ+4AKzk4HfjNM1fd03CPCHvnsZ9k49NvWPLPdnxlOgrK1bnwTKnCqsedBNYid53DdVQ23XBJE+JvqaqIK8KzRnvxPVcR1XdxOGepqB6nRSwdtTvzBvv5twNF95wGVXF3+E7PKgmWi7Rk7XGM3ueNw8WS8i8u8Gcg7RhqqvA+nKKKzUS8T39sO83D2pJ3x9PyDP6k7tMglXfYyobOMcdpEIx9ksL3QTWlRQNFVw633PDf9GDyxYk/724rcQMVXdsuRs/RT6kM7xTXSQI3bHRNVThGUm2PMpLPDd9VwfLc+N3IyXHk/937sn5WYP12sE3TUo+x4PQQmkzfaXuoOro3yfndcW6u7z33BanaqB777oldmXwYSOxSPqgTZpOANG85HM5bUYo0S/c5Q6w+n1Xgya6/80Xk6nSu///ntOS/HbtvvbswDrdYXZBl+OJfsKpyV9RnaqWFSmrpI3rrqZK76GPUyabHfWfG6Xqwe2PRGNma5pzPDVZtJii5HtKU1V+W7KfPVpX47ntB8OB5bnl+uj5lCq88YffMhbMqV5vUuy5X0whukw9+7i6VB38Cy2SpFP8Pt/Ye4alaqD7fn1nLCm9MRAJ3P3pLwWp+vnmOzv9ZucGiSaR3vC1yqqAtbdgtXmxwqGbFEdlYMak9FTpPfDe5JTgQleNIFjTimaYF7sVBauyriatxJ7s5MxWZF+JqqGbCSnLjP27RajSe3j3jtRQz/qMVOFWNZk4qjf0oNwSe4m+VmlgYfdhwmea2ULawT9TjqoT30yufY21t3pH3ygOlvf2Hw0J9Ty95zlLMxriutrvlxgl1JTdFER0G6OT/6MYWH+khTpSQjfPPCJOVdzYempscqinN9h7+yd3DEfV/YUP+sI+8wLHNcCmIBLBUnQbI6p0hKKhjpdUDOmXt6ess5YvNEpVPNSzfFHydU60Fl+Tqp9t/kVpm5yEJ1ZLcrejlbu/re1bRdpTtioauhmbGropHQzdGO2EekqLkq825+S8q8FaV/2RLHtIVrQ42XJXcatIYS9ItH/vK1VCN4OWb6y31/vL27OkSje2xvyE+M/gUEWHhnreOvq2rJFDbREllDh26m5uq4lvx61IKunQ0M04M1TFQz2vpYR62ryDoZ6XDrzqDctRNPoBlhLXg5ri23GL7m/bucV5qpqXpYRuNq03T1U81LMpRdkV1m2HqlcOvCbFYE+skzhiTZPalD5AQPDIk34FdORvHA3d3JIauqlNF7ox2gn11KQc1/hhZQFOeYlCxn0lFwLVjzwx4As+/6wqoZtg0PfofdKo0r0PD43RAn7Nmoa3SfoJ4VxTbmhIc+pYOa23yDGweotTQjczp8ilKh7q+d07VlDFdIVTVGKeTPxrrv6xcvpBmEKJHPi77afqQkrGZvjgHiuoiod69uyUDhbWlfTpyh0HYRrQhpjPDodu2gyFbgx7H35+Q6ylWaJgdUtI1uazyXrQBUf3asKHjWtRv3b0qzaC9Ycpm3/Eov4XnrCOqrj34dlHtUhElmBZCcpK70xuBJ7dcdi4vmwhRJr5pQ2T7QOrc1tK6Gbl+1ZTFQ/1rFosRap4nnG1S1dYDBODJdHhbiFY2kAKPA43gcm9M8MJKBaFbk6dzD50k02o54R5MwibXVZu5xC/qBWzoFVgYQdgDQh9D0iEsjx089WhoZv7breJKt3YuvensR5TuoaSLIk1XkM8+NKNNgvB0gYqDYVMeDaftdrSSj3yyT/+JTup+iLU87yWrbphJYgZZIW2wmyXWOxqE1jM20QJxKbjeRZS1TQnRfd+8nv7qYqHerZsyC5KJjdNdEgYp6+vz2VgaQPngWUuj9YG0kqLvmZN6OZmC0M3RvutQ0M9IhYFVAnJ0HhjJSjxrC9bwcILL5pDTbad/NDNn6dswW1B6MZwqOfhe8VDPXitMNgl1kpc7WWQ7m23CSwDk5ZmwX5/vSlpd4FZU52lKu59mDVNRBik3SE6uRlXQ6YriZsfOQAW3wnR+RbXg8SUh9pXU8bp4F4VqIqHevbuzEgVHma5+exDGoNihVPUPrD0SUt0ym1bKyl0842hoZtf3KQOWP2hntaWNBoQiVlKFSaKpdaVTWCx7sBWEHUZS1CIhG7aUkI3Lz6pDlVx78Nzj2nXEoheUG6RxzLhu2I4rFsM2geWHuQSNRcwt5lvTIVuUvbjD61aohpVce/DqiXX9Cxk3g3KXINaK0LOzoCFG0YoszQu4NP9uXjZUVX5qxRKT5fZF7rJItRzcjAzDP+n0OZ1MgbCIq+YA2BpA9WVBhKJsjsnbEjops/u0I3JUI8VvvWrQ0PWrTSdAQupYZNiNgqHhMYZBsuXAm7g7ddVpiqeaDrhFc2uhvAZAhvwtRWshA0hvEtYzJghnxq6CW/ZqD5Vce/Dpx/bIHzELr0EQxWwtIF0GgNRTzIBBU+95zDV5INJKBKc8LJbwOJWNVuUoKWLTYfBYh7Gg5J5U5rBRd1lrfjbGag68Mea7xolvOF9u4ZsSqucjTXmJ9ykDWJnqw/EbpsSdAAs7YtULaH80jggV7TS69KB1fjusBx7PcH35zgZeB6u3/bfwcVzY7YoJo6usC7pSiGwaLhqcANGxJPB2W6ZU3evrQRv1GIZrhPrvhJcMo/kAiWQ+vF1FJzFOjvsETVCRtQG1kyuBotGyMJYitk12Tryt1pQdLsBoijBRbO9o29wTPGNviH4/uxYW4udcsY1LZQjnjNgoe9J2zC2CwXlyxX3JIH1Je3yNsMf7PeFt27yPX6/rdkyT9wf/myTdbXRaZzsBLNtNq0cBktfADNLG4xgxLT6t/uRAiwcXWbIbmlmo3aG3EKeHhpD3CbaWOeIeFF/iFfiFvCuAUsbqD3CrjRcfNK+Xiv9fr8zQsrc2dwU/mxzYNLrHGQiQd/dfWtg8htckMs6KFhizHit7DfYVQFLFwFssXIxiEPEipshwSZy7FBozQrg8D/zaP8+bD++Lo0ZTsiIklReHFq7MnLsMG/XFGh4QRGpDfkLSoOlDWQziha4OmAMRikXQ29Gmy5Gz1b196aL/MoftVhUwfvV0+QtzTl2DVi6A4IkIWen7hxoOlWOOBcUBUsbOL0DthSdt9zQ9GJ564ov3AqWvpDBOLAzUJozDaHZkCToVrC0geILFQxPdzWW1QjN6uIId4Ol2ZWhmzMNQan5VRyloLD0U2Lk7g6dkw3fOl5QNRc9o9QUGS5jYj7EEyORSB6gqxtiQThEbBz0rbsSLG0gnkismm9kfqk4pOFPRiwIx6k4oLvBSrYhDOQG5npDFAhEEWeVi8HSBoqWUIsU+YgWkOVoQ+uh/sgFdYUneZQrZMqcT768Om5l+xsPzuNjraus/twHVsK24PvK1GVPyaUijXmaiYrKLXe5jke5S8p8Xym5xMjAGSF9C2HVGg/IY/KwPLJbJiq3gpWwuki6xdelQhjfOt3HA7LPgkun51HuFT3u5uqBZvWWBzY3fVc0nsvVoa1Rbh8GJi2GgWVjDuAFUjwIj6Na4G8kgpVQHHzF2QaYvBHXmSPYUtw2N88j5IxyzxGwEsqRNSPWLjVPrvDXc5PcKjfMbedYTkdOgaU3/Ie604tQGn5qBaNp3BI3htbjJrnVnEydzUGwEo4JLBVWVRzGx0GPDKTjyytugNvgZrglbozbc53WzoOVYsFguDCQaBzsGFQPZr5tSRN8EB/Hh/LR3AC3wc3kvAduRICV3HBe43JkzigrK2Px1djYyBRCaqFEzrgUF+SyXByY+CA+jg8daSnXIwusZEXJSLO/T0NDA2sxhh9vJDYZvwIBKRXoKfjA+sEeigy0BDc0/sh/8QJexot5C2/k7VyES3FBfuXifEQOK7s8WKImP8njLPsJ9HKkO+E5jGvSntBfmETgUjrQ+IFf+SP/paci8mLewht5e76CLdH+H5nMNBUt+3p0AAAAAElFTkSuQmCC", - "description": "Displays the latest values of the attributes or time series data for multiple entities in a polar area chart. Supports numeric values only.", + "image": "tb-image:cG9sYXItYXJlYS5zdmc=:IlBvbGFyIGFyZWEiIHN5c3RlbSB3aWRnZXQgaW1hZ2U=;data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjAwIiBoZWlnaHQ9IjE2MCIgdmlld0JveD0iMCAwIDIwMCAxNjAiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxyZWN0IHdpZHRoPSIyMDAiIGhlaWdodD0iMTYwIiByeD0iNCIgZmlsbD0id2hpdGUiLz4KPGNpcmNsZSBjeD0iOTkuNDc3NiIgY3k9IjgwIiByPSI2OS43Mzg4IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC4zOCIgc3Ryb2tlLXdpZHRoPSIwLjUyMjM4OCIvPgo8Y2lyY2xlIGN4PSI5OS40Nzc2IiBjeT0iODAiIHI9IjUzLjAyMjQiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS1vcGFjaXR5PSIwLjEyIiBzdHJva2Utd2lkdGg9IjAuNTIyMzg4Ii8+CjxjaXJjbGUgY3g9Ijk5LjQ3NzYiIGN5PSI4MCIgcj0iMzYuMzA2IiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC4xMiIgc3Ryb2tlLXdpZHRoPSIwLjUyMjM4OCIvPgo8Y2lyY2xlIGN4PSI5OS40Nzc2IiBjeT0iODAiIHI9IjE5LjU4OTYiIHN0cm9rZT0iYmxhY2siIHN0cm9rZS1vcGFjaXR5PSIwLjEyIiBzdHJva2Utd2lkdGg9IjAuNTIyMzg4Ii8+CjxjaXJjbGUgY3g9Ijk5LjQ3NzYiIGN5PSI4MC4wMDAxIiByPSIyLjg3MzEzIiBzdHJva2U9ImJsYWNrIiBzdHJva2Utb3BhY2l0eT0iMC4xMiIgc3Ryb2tlLXdpZHRoPSIwLjUyMjM4OCIvPgo8cGF0aCBkPSJNOTkuNDc3NiAxMC41MjIzVjgwIiBzdHJva2U9IiM5RTlFOUUiIHN0cm9rZS13aWR0aD0iMC41MjIzODgiLz4KPHBhdGggZD0iTTk5LjQ3NzYgMjYuNzE2NEg5Ny4zODgxIiBzdHJva2U9IiM5RTlFOUUiIHN0cm9rZS13aWR0aD0iMC41MjIzODgiLz4KPHBhdGggZD0iTTk5LjQ3NzYgNDMuNDMyOUg5Ny4zODgxIiBzdHJva2U9IiM5RTlFOUUiIHN0cm9rZS13aWR0aD0iMC41MjIzODgiLz4KPHBhdGggZD0iTTk5LjQ3NzYgNjAuMTQ5M0g5Ny4zODgxIiBzdHJva2U9IiM5RTlFOUUiIHN0cm9rZS13aWR0aD0iMC41MjIzODgiLz4KPHJlY3Qgd2lkdGg9IjE1IiBoZWlnaHQ9IjguMzU4MjEiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDgyLjc2MTIgMzkuMjUzOCkiIGZpbGw9IndoaXRlIi8+CjxwYXRoIGQ9Ik04NS45MjI5IDM5Ljk1OFY0NS45MzI5SDg1LjE2NzlWNDAuOTAwOEw4My42NDU2IDQxLjQ1NThWNDAuNzc0M0w4NS44MDQ2IDM5Ljk1OEg4NS45MjI5Wk05Mi4xMzQ1IDQyLjQ4ODRWNDMuMzk0NEM5Mi4xMzQ1IDQzLjg4MTQgOTIuMDkwOSA0NC4yOTIyIDkyLjAwMzkgNDQuNjI2OUM5MS45MTY4IDQ0Ljk2MTUgOTEuNzkxNiA0NS4yMzA5IDkxLjYyODQgNDUuNDM1QzkxLjQ2NTEgNDUuNjM5IDkxLjI2NzkgNDUuNzg3MyA5MS4wMzY2IDQ1Ljg3OThDOTAuODA4MSA0NS45Njk2IDkwLjU0OTYgNDYuMDE0NSA5MC4yNjEyIDQ2LjAxNDVDOTAuMDMyNyA0Ni4wMTQ1IDg5LjgyMTggNDUuOTg1OSA4OS42Mjg2IDQ1LjkyODhDODkuNDM1NCA0NS44NzE2IDg5LjI2MTMgNDUuNzgwNSA4OS4xMDYyIDQ1LjY1NTNDODguOTUzOSA0NS41Mjc1IDg4LjgyMzMgNDUuMzYxNSA4OC43MTQ0IDQ1LjE1NzRDODguNjA1NiA0NC45NTM0IDg4LjUyMjYgNDQuNzA1OCA4OC40NjU1IDQ0LjQxNDdDODguNDA4NCA0NC4xMjM1IDg4LjM3OTggNDMuNzgzNSA4OC4zNzk4IDQzLjM5NDRWNDIuNDg4NEM4OC4zNzk4IDQyLjAwMTMgODguNDIzMyA0MS41OTMyIDg4LjUxMDQgNDEuMjY0Qzg4LjYwMDIgNDAuOTM0OCA4OC43MjY3IDQwLjY3MDkgODguODg5OSA0MC40NzIzQzg5LjA1MzIgNDAuMjcwOSA4OS4yNDkxIDQwLjEyNjcgODkuNDc3NiA0MC4wMzk3Qzg5LjcwODkgMzkuOTUyNiA4OS45Njc0IDM5LjkwOTEgOTAuMjUzIDM5LjkwOTFDOTAuNDg0MyAzOS45MDkxIDkwLjY5NjUgMzkuOTM3NiA5MC44ODk3IDM5Ljk5NDhDOTEuMDg1NiA0MC4wNDkyIDkxLjI1OTcgNDAuMTM3NiA5MS40MTIxIDQwLjI2MDFDOTEuNTY0NCA0MC4zNzk4IDkxLjY5MzcgNDAuNTQwMyA5MS43OTk4IDQwLjc0MTZDOTEuOTA4NiA0MC45NDAyIDkxLjk5MTYgNDEuMTgzOCA5Mi4wNDg3IDQxLjQ3MjJDOTIuMTA1OSA0MS43NjA2IDkyLjEzNDUgNDIuMDk5MyA5Mi4xMzQ1IDQyLjQ4ODRaTTkxLjM3NTQgNDMuNTE2OFY0Mi4zNjE4QzkxLjM3NTQgNDIuMDk1MiA5MS4zNTkgNDEuODYxMiA5MS4zMjY0IDQxLjY1OTlDOTEuMjk2NSA0MS40NTU4IDkxLjI1MTYgNDEuMjgxNyA5MS4xOTE3IDQxLjEzNzVDOTEuMTMxOCA0MC45OTMzIDkxLjA1NTcgNDAuODc2MyA5MC45NjMyIDQwLjc4NjVDOTAuODczNCA0MC42OTY3IDkwLjc2ODYgNDAuNjMxNCA5MC42NDg5IDQwLjU5MDZDOTAuNTMxOSA0MC41NDcxIDkwLjQgNDAuNTI1MyA5MC4yNTMgNDAuNTI1M0M5MC4wNzM1IDQwLjUyNTMgODkuOTE0MyA0MC41NTkzIDg5Ljc3NTUgNDAuNjI3NEM4OS42MzY4IDQwLjY5MjcgODkuNTE5OCA0MC43OTc0IDg5LjQyNDYgNDAuOTQxNkM4OS4zMzIxIDQxLjA4NTggODkuMjYxMyA0MS4yNzQ5IDg5LjIxMjMgNDEuNTA4OUM4OS4xNjM0IDQxLjc0MjkgODkuMTM4OSA0Mi4wMjcyIDg5LjEzODkgNDIuMzYxOFY0My41MTY4Qzg5LjEzODkgNDMuNzgzNSA4OS4xNTM4IDQ0LjAxODggODkuMTgzOCA0NC4yMjI5Qzg5LjIxNjQgNDQuNDI2OSA4OS4yNjQgNDQuNjAzOCA4OS4zMjY2IDQ0Ljc1MzRDODkuMzg5MiA0NC45MDAzIDg5LjQ2NTQgNDUuMDIxNCA4OS41NTUyIDQ1LjExNjZDODkuNjQ0OSA0NS4yMTE5IDg5Ljc0ODMgNDUuMjgyNiA4OS44NjUzIDQ1LjMyODlDODkuOTg1IDQ1LjM3MjQgOTAuMTE3IDQ1LjM5NDEgOTAuMjYxMiA0NS4zOTQxQzkwLjQ0NjIgNDUuMzk0MSA5MC42MDgxIDQ1LjM1ODggOTAuNzQ2OSA0NS4yODhDOTAuODg1NiA0NS4yMTczIDkxLjAwMTIgNDUuMTA3MSA5MS4wOTM4IDQ0Ljk1NzVDOTEuMTg5IDQ0LjgwNTEgOTEuMjU5NyA0NC42MTA2IDkxLjMwNiA0NC4zNzM5QzkxLjM1MjIgNDQuMTM0NCA5MS4zNzU0IDQzLjg0ODggOTEuMzc1NCA0My41MTY4Wk05Ny4wOTcxIDQyLjQ4ODRWNDMuMzk0NEM5Ny4wOTcxIDQzLjg4MTQgOTcuMDUzNiA0NC4yOTIyIDk2Ljk2NjUgNDQuNjI2OUM5Ni44Nzk1IDQ0Ljk2MTUgOTYuNzU0MyA0NS4yMzA5IDk2LjU5MTEgNDUuNDM1Qzk2LjQyNzggNDUuNjM5IDk2LjIzMDYgNDUuNzg3MyA5NS45OTkzIDQ1Ljg3OThDOTUuNzcwOCA0NS45Njk2IDk1LjUxMjMgNDYuMDE0NSA5NS4yMjM5IDQ2LjAxNDVDOTQuOTk1MyA0Ni4wMTQ1IDk0Ljc4NDUgNDUuOTg1OSA5NC41OTEzIDQ1LjkyODhDOTQuMzk4MSA0NS44NzE2IDk0LjIyNCA0NS43ODA1IDk0LjA2ODkgNDUuNjU1M0M5My45MTY2IDQ1LjUyNzUgOTMuNzg2IDQ1LjM2MTUgOTMuNjc3MSA0NS4xNTc0QzkzLjU2ODMgNDQuOTUzNCA5My40ODUzIDQ0LjcwNTggOTMuNDI4MiA0NC40MTQ3QzkzLjM3MSA0NC4xMjM1IDkzLjM0MjUgNDMuNzgzNSA5My4zNDI1IDQzLjM5NDRWNDIuNDg4NEM5My4zNDI1IDQyLjAwMTMgOTMuMzg2IDQxLjU5MzIgOTMuNDczMSA0MS4yNjRDOTMuNTYyOSA0MC45MzQ4IDkzLjY4OTQgNDAuNjcwOSA5My44NTI2IDQwLjQ3MjNDOTQuMDE1OSA0MC4yNzA5IDk0LjIxMTggNDAuMTI2NyA5NC40NDAzIDQwLjAzOTdDOTQuNjcxNiAzOS45NTI2IDk0LjkzIDM5LjkwOTEgOTUuMjE1NyAzOS45MDkxQzk1LjQ0NyAzOS45MDkxIDk1LjY1OTIgMzkuOTM3NiA5NS44NTI0IDM5Ljk5NDhDOTYuMDQ4MyA0MC4wNDkyIDk2LjIyMjQgNDAuMTM3NiA5Ni4zNzQ4IDQwLjI2MDFDOTYuNTI3MSA0MC4zNzk4IDk2LjY1NjQgNDAuNTQwMyA5Ni43NjI1IDQwLjc0MTZDOTYuODcxMyA0MC45NDAyIDk2Ljk1NDMgNDEuMTgzOCA5Ny4wMTE0IDQxLjQ3MjJDOTcuMDY4NiA0MS43NjA2IDk3LjA5NzEgNDIuMDk5MyA5Ny4wOTcxIDQyLjQ4ODRaTTk2LjMzOCA0My41MTY4VjQyLjM2MThDOTYuMzM4IDQyLjA5NTIgOTYuMzIxNyA0MS44NjEyIDk2LjI4OTEgNDEuNjU5OUM5Ni4yNTkxIDQxLjQ1NTggOTYuMjE0MiA0MS4yODE3IDk2LjE1NDQgNDEuMTM3NUM5Ni4wOTQ1IDQwLjk5MzMgOTYuMDE4NCA0MC44NzYzIDk1LjkyNTggNDAuNzg2NUM5NS44MzYxIDQwLjY5NjcgOTUuNzMxMyA0MC42MzE0IDk1LjYxMTYgNDAuNTkwNkM5NS40OTQ2IDQwLjU0NzEgOTUuMzYyNiA0MC41MjUzIDk1LjIxNTcgNDAuNTI1M0M5NS4wMzYyIDQwLjUyNTMgOTQuODc3IDQwLjU1OTMgOTQuNzM4MiA0MC42Mjc0Qzk0LjU5OTUgNDAuNjkyNyA5NC40ODI1IDQwLjc5NzQgOTQuMzg3MiA0MC45NDE2Qzk0LjI5NDcgNDEuMDg1OCA5NC4yMjQgNDEuMjc0OSA5NC4xNzUgNDEuNTA4OUM5NC4xMjYxIDQxLjc0MjkgOTQuMTAxNiA0Mi4wMjcyIDk0LjEwMTYgNDIuMzYxOFY0My41MTY4Qzk0LjEwMTYgNDMuNzgzNSA5NC4xMTY1IDQ0LjAxODggOTQuMTQ2NSA0NC4yMjI5Qzk0LjE3OTEgNDQuNDI2OSA5NC4yMjY3IDQ0LjYwMzggOTQuMjg5MyA0NC43NTM0Qzk0LjM1MTkgNDQuOTAwMyA5NC40MjgxIDQ1LjAyMTQgOTQuNTE3OCA0NS4xMTY2Qzk0LjYwNzYgNDUuMjExOSA5NC43MTEgNDUuMjgyNiA5NC44MjggNDUuMzI4OUM5NC45NDc3IDQ1LjM3MjQgOTUuMDc5NyA0NS4zOTQxIDk1LjIyMzkgNDUuMzk0MUM5NS40MDg5IDQ1LjM5NDEgOTUuNTcwOCA0NS4zNTg4IDk1LjcwOTUgNDUuMjg4Qzk1Ljg0ODMgNDUuMjE3MyA5NS45NjM5IDQ1LjEwNzEgOTYuMDU2NCA0NC45NTc1Qzk2LjE1MTcgNDQuODA1MSA5Ni4yMjI0IDQ0LjYxMDYgOTYuMjY4NyA0NC4zNzM5Qzk2LjMxNDkgNDQuMTM0NCA5Ni4zMzggNDMuODQ4OCA5Ni4zMzggNDMuNTE2OFoiIGZpbGw9ImJsYWNrIiBmaWxsLW9wYWNpdHk9IjAuNzYiLz4KPHJlY3Qgd2lkdGg9IjEwIiBoZWlnaHQ9IjguMzU4MjEiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDg2Ljk0MDMgNTUuOTcwMSkiIGZpbGw9IndoaXRlIi8+CjxwYXRoIGQ9Ik04OC41NTcgNTkuODIwOUw4Ny45NTMgNTkuNjY1OEw4OC4yNTA5IDU2LjcwN0g5MS4yOTk2VjU3LjQwNDlIODguODkxN0w4OC43MTIxIDU5LjAyMUM4OC44MjA5IDU4Ljk1ODQgODguOTU4MyA1OC44OTk5IDg5LjEyNDMgNTguODQ1NUM4OS4yOTMgNTguNzkxMSA4OS40ODYyIDU4Ljc2MzkgODkuNzAzOCA1OC43NjM5Qzg5Ljk3ODYgNTguNzYzOSA5MC4yMjQ5IDU4LjgxMTUgOTAuNDQyNSA1OC45MDY3QzkwLjY2MDIgNTguOTk5MyA5MC44NDUyIDU5LjEzMjYgOTAuOTk3NiA1OS4zMDY3QzkxLjE1MjYgNTkuNDgwOCA5MS4yNzEgNTkuNjkwMyA5MS4zNTI2IDU5LjkzNTJDOTEuNDM0MiA2MC4xODAxIDkxLjQ3NTEgNjAuNDUzNSA5MS40NzUxIDYwLjc1NTVDOTEuNDc1MSA2MS4wNDEyIDkxLjQzNTYgNjEuMzAzNyA5MS4zNTY3IDYxLjU0MzJDOTEuMjgwNSA2MS43ODI2IDkxLjE2NDkgNjEuOTkyMSA5MS4wMDk4IDYyLjE3MTdDOTAuODU0NyA2Mi4zNDg1IDkwLjY1ODggNjIuNDg1OSA5MC40MjIxIDYyLjU4MzlDOTAuMTg4MSA2Mi42ODE4IDg5LjkxMiA2Mi43MzA4IDg5LjU5MzYgNjIuNzMwOEM4OS4zNTQyIDYyLjczMDggODkuMTI3IDYyLjY5ODEgODguOTEyMSA2Mi42MzI4Qzg4LjY5OTkgNjIuNTY0OCA4OC41MDk0IDYyLjQ2MjggODguMzQwNyA2Mi4zMjY4Qzg4LjE3NDggNjIuMTg4IDg4LjAzODcgNjIuMDE2NiA4Ny45MzI2IDYxLjgxMjVDODcuODI5MiA2MS42MDU4IDg3Ljc2MzkgNjEuMzYzNiA4Ny43MzY3IDYxLjA4NjFIODguNDU1Qzg4LjQ4NzYgNjEuMzA5MiA4OC41NTI5IDYxLjQ5NjkgODguNjUwOSA2MS42NDkzQzg4Ljc0ODggNjEuODAxNiA4OC44NzY3IDYxLjkxNzMgODkuMDM0NSA2MS45OTYyQzg5LjE5NSA2Mi4wNzI0IDg5LjM4MTQgNjIuMTEwNSA4OS41OTM2IDYyLjExMDVDODkuNzczMiA2Mi4xMTA1IDg5LjkzMjQgNjIuMDc5MiA5MC4wNzExIDYyLjAxNjZDOTAuMjA5OSA2MS45NTQgOTAuMzI2OSA2MS44NjQyIDkwLjQyMjEgNjEuNzQ3MkM5MC41MTczIDYxLjYzMDIgOTAuNTg5NCA2MS40ODg4IDkwLjYzODQgNjEuMzIyOEM5MC42OTAxIDYxLjE1NjggOTAuNzE2IDYwLjk3MDUgOTAuNzE2IDYwLjc2MzdDOTAuNzE2IDYwLjU3NTkgOTAuNjkwMSA2MC40MDE4IDkwLjYzODQgNjAuMjQxM0M5MC41ODY3IDYwLjA4MDggOTAuNTA5MiA1OS45NDA2IDkwLjQwNTggNTkuODIwOUM5MC4zMDUxIDU5LjcwMTIgOTAuMTgxMyA1OS42MDg3IDkwLjAzNDQgNTkuNTQzNEM4OS44ODc1IDU5LjQ3NTQgODkuNzE4OCA1OS40NDE0IDg5LjUyODMgNTkuNDQxNEM4OS4yNzUzIDU5LjQ0MTQgODkuMDgzNSA1OS40NzU0IDg4Ljk1MjkgNTkuNTQzNEM4OC44MjUgNTkuNjExNCA4OC42OTMxIDU5LjcwMzkgODguNTU3IDU5LjgyMDlaTTk2LjI5NDkgNTkuMjA0N1Y2MC4xMTA3Qzk2LjI5NDkgNjAuNTk3NyA5Ni4yNTE0IDYxLjAwODUgOTYuMTY0MyA2MS4zNDMyQzk2LjA3NzIgNjEuNjc3OSA5NS45NTIxIDYxLjk0NzIgOTUuNzg4OCA2Mi4xNTEzQzk1LjYyNTYgNjIuMzU1MyA5NS40MjgzIDYyLjUwMzYgOTUuMTk3MSA2Mi41OTYxQzk0Ljk2ODUgNjIuNjg1OSA5NC43MTAxIDYyLjczMDggOTQuNDIxNyA2Mi43MzA4Qzk0LjE5MzEgNjIuNzMwOCA5My45ODIyIDYyLjcwMjIgOTMuNzg5MSA2Mi42NDUxQzkzLjU5NTkgNjIuNTg4IDkzLjQyMTggNjIuNDk2OCA5My4yNjY3IDYyLjM3MTdDOTMuMTE0MyA2Mi4yNDM4IDkyLjk4MzcgNjIuMDc3OCA5Mi44NzQ5IDYxLjg3MzhDOTIuNzY2MSA2MS42Njk3IDkyLjY4MzEgNjEuNDIyMSA5Mi42MjU5IDYxLjEzMUM5Mi41Njg4IDYwLjgzOTkgOTIuNTQwMiA2MC40OTk4IDkyLjU0MDIgNjAuMTEwN1Y1OS4yMDQ3QzkyLjU0MDIgNTguNzE3NyA5Mi41ODM4IDU4LjMwOTUgOTIuNjcwOCA1Ny45ODAzQzkyLjc2MDYgNTcuNjUxMSA5Mi44ODcxIDU3LjM4NzIgOTMuMDUwNCA1Ny4xODg2QzkzLjIxMzYgNTYuOTg3MiA5My40MDk1IDU2Ljg0MyA5My42MzgxIDU2Ljc1NkM5My44NjkzIDU2LjY2ODkgOTQuMTI3OCA1Ni42MjU0IDk0LjQxMzUgNTYuNjI1NEM5NC42NDQ4IDU2LjYyNTQgOTQuODU3IDU2LjY1NCA5NS4wNTAxIDU2LjcxMTFDOTUuMjQ2IDU2Ljc2NTUgOTUuNDIwMiA1Ni44NTM5IDk1LjU3MjUgNTYuOTc2NEM5NS43MjQ5IDU3LjA5NjEgOTUuODU0MSA1Ny4yNTY2IDk1Ljk2MDIgNTcuNDU3OUM5Ni4wNjkxIDU3LjY1NjYgOTYuMTUyMSA1Ny45MDAxIDk2LjIwOTIgNTguMTg4NUM5Ni4yNjYzIDU4LjQ3NjkgOTYuMjk0OSA1OC44MTU2IDk2LjI5NDkgNTkuMjA0N1pNOTUuNTM1OCA2MC4yMzMxVjU5LjA3ODJDOTUuNTM1OCA1OC44MTE1IDk1LjUxOTUgNTguNTc3NSA5NS40ODY4IDU4LjM3NjJDOTUuNDU2OSA1OC4xNzIxIDk1LjQxMiA1Ny45OTggOTUuMzUyMiA1Ny44NTM4Qzk1LjI5MjMgNTcuNzA5NiA5NS4yMTYxIDU3LjU5MjYgOTUuMTIzNiA1Ny41MDI4Qzk1LjAzMzggNTcuNDEzIDk0LjkyOTEgNTcuMzQ3NyA5NC44MDk0IDU3LjMwNjlDOTQuNjkyNCA1Ny4yNjM0IDk0LjU2MDQgNTcuMjQxNiA5NC40MTM1IDU3LjI0MTZDOTQuMjMzOSA1Ny4yNDE2IDk0LjA3NDggNTcuMjc1NiA5My45MzYgNTcuMzQzN0M5My43OTcyIDU3LjQwOSA5My42ODAyIDU3LjUxMzcgOTMuNTg1IDU3LjY1NzlDOTMuNDkyNSA1Ny44MDIxIDkzLjQyMTggNTcuOTkxMiA5My4zNzI4IDU4LjIyNTJDOTMuMzIzOCA1OC40NTkyIDkzLjI5OTMgNTguNzQzNSA5My4yOTkzIDU5LjA3ODJWNjAuMjMzMUM5My4yOTkzIDYwLjQ5OTggOTMuMzE0MyA2MC43MzUxIDkzLjM0NDIgNjAuOTM5MkM5My4zNzY5IDYxLjE0MzIgOTMuNDI0NSA2MS4zMjAxIDkzLjQ4NzEgNjEuNDY5N0M5My41NDk2IDYxLjYxNjYgOTMuNjI1OCA2MS43Mzc3IDkzLjcxNTYgNjEuODMyOUM5My44MDU0IDYxLjkyODIgOTMuOTA4OCA2MS45OTg5IDk0LjAyNTggNjIuMDQ1MkM5NC4xNDU1IDYyLjA4ODcgOTQuMjc3NSA2Mi4xMTA1IDk0LjQyMTcgNjIuMTEwNUM5NC42MDY3IDYyLjExMDUgOTQuNzY4NSA2Mi4wNzUxIDk0LjkwNzMgNjIuMDA0M0M5NS4wNDYxIDYxLjkzMzYgOTUuMTYxNyA2MS44MjM0IDk1LjI1NDIgNjEuNjczOEM5NS4zNDk0IDYxLjUyMTQgOTUuNDIwMiA2MS4zMjY5IDk1LjQ2NjQgNjEuMDkwMkM5NS41MTI3IDYwLjg1MDcgOTUuNTM1OCA2MC41NjUxIDk1LjUzNTggNjAuMjMzMVoiIGZpbGw9ImJsYWNrIiBmaWxsLW9wYWNpdHk9IjAuNzYiLz4KPHJlY3Qgd2lkdGg9IjUiIGhlaWdodD0iOC4zNTgyMSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoOTIuMTY0MiA3Mi42ODY1KSIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTk2LjY4NjcgNzUuOTIxMVY3Ni44MjcxQzk2LjY4NjcgNzcuMzE0MSA5Ni42NDMyIDc3LjcyNSA5Ni41NTYxIDc4LjA1OTZDOTYuNDY5IDc4LjM5NDMgOTYuMzQzOSA3OC42NjM2IDk2LjE4MDYgNzguODY3N0M5Ni4wMTc0IDc5LjA3MTggOTUuODIwMSA3OS4yMiA5NS41ODg5IDc5LjMxMjVDOTUuMzYwMyA3OS40MDIzIDk1LjEwMTggNzkuNDQ3MiA5NC44MTM0IDc5LjQ0NzJDOTQuNTg0OSA3OS40NDcyIDk0LjM3NCA3OS40MTg3IDk0LjE4MDkgNzkuMzYxNUM5My45ODc3IDc5LjMwNDQgOTMuODEzNiA3OS4yMTMyIDkzLjY1ODUgNzkuMDg4MUM5My41MDYxIDc4Ljk2MDIgOTMuMzc1NSA3OC43OTQyIDkzLjI2NjcgNzguNTkwMkM5My4xNTc4IDc4LjM4NjEgOTMuMDc0OSA3OC4xMzg1IDkzLjAxNzcgNzcuODQ3NEM5Mi45NjA2IDc3LjU1NjMgOTIuOTMyIDc3LjIxNjIgOTIuOTMyIDc2LjgyNzFWNzUuOTIxMUM5Mi45MzIgNzUuNDM0MSA5Mi45NzU2IDc1LjAyNiA5My4wNjI2IDc0LjY5NjhDOTMuMTUyNCA3NC4zNjc1IDkzLjI3ODkgNzQuMTAzNiA5My40NDIyIDczLjkwNUM5My42MDU0IDczLjcwMzcgOTMuODAxMyA3My41NTk1IDk0LjAyOTkgNzMuNDcyNEM5NC4yNjExIDczLjM4NTMgOTQuNTE5NiA3My4zNDE4IDk0LjgwNTMgNzMuMzQxOEM5NS4wMzY1IDczLjM0MTggOTUuMjQ4OCA3My4zNzA0IDk1LjQ0MTkgNzMuNDI3NUM5NS42Mzc4IDczLjQ4MTkgOTUuODEyIDczLjU3MDQgOTUuOTY0MyA3My42OTI4Qzk2LjExNjcgNzMuODEyNSA5Ni4yNDU5IDczLjk3MyA5Ni4zNTIgNzQuMTc0NEM5Ni40NjA5IDc0LjM3MyA5Ni41NDM4IDc0LjYxNjUgOTYuNjAxIDc0LjkwNDlDOTYuNjU4MSA3NS4xOTMzIDk2LjY4NjcgNzUuNTMyIDk2LjY4NjcgNzUuOTIxMVpNOTUuOTI3NiA3Ni45NDk2Vjc1Ljc5NDZDOTUuOTI3NiA3NS41MjggOTUuOTExMyA3NS4yOTQgOTUuODc4NiA3NS4wOTI2Qzk1Ljg0ODcgNzQuODg4NiA5NS44MDM4IDc0LjcxNDQgOTUuNzQzOSA3NC41NzAyQzk1LjY4NDEgNzQuNDI2IDk1LjYwNzkgNzQuMzA5IDk1LjUxNTQgNzQuMjE5M0M5NS40MjU2IDc0LjEyOTUgOTUuMzIwOSA3NC4wNjQyIDk1LjIwMTEgNzQuMDIzNEM5NS4wODQyIDczLjk3OTggOTQuOTUyMiA3My45NTgxIDk0LjgwNTMgNzMuOTU4MUM5NC42MjU3IDczLjk1ODEgOTQuNDY2NSA3My45OTIxIDk0LjMyNzggNzQuMDYwMUM5NC4xODkgNzQuMTI1NCA5NC4wNzIgNzQuMjMwMSA5My45NzY4IDc0LjM3NDNDOTMuODg0MyA3NC41MTg1IDkzLjgxMzYgNzQuNzA3NiA5My43NjQ2IDc0Ljk0MTZDOTMuNzE1NiA3NS4xNzU2IDkzLjY5MTEgNzUuNDU5OSA5My42OTExIDc1Ljc5NDZWNzYuOTQ5NkM5My42OTExIDc3LjIxNjIgOTMuNzA2MSA3Ny40NTE1IDkzLjczNiA3Ny42NTU2QzkzLjc2ODcgNzcuODU5NyA5My44MTYzIDc4LjAzNjUgOTMuODc4OSA3OC4xODYxQzkzLjk0MTQgNzguMzMzMSA5NC4wMTc2IDc4LjQ1NDEgOTQuMTA3NCA3OC41NDk0Qzk0LjE5NzIgNzguNjQ0NiA5NC4zMDA2IDc4LjcxNTMgOTQuNDE3NiA3OC43NjE2Qzk0LjUzNzMgNzguODA1MSA5NC42NjkyIDc4LjgyNjkgOTQuODEzNCA3OC44MjY5Qzk0Ljk5ODUgNzguODI2OSA5NS4xNjAzIDc4Ljc5MTUgOTUuMjk5MSA3OC43MjA4Qzk1LjQzNzkgNzguNjUgOTUuNTUzNSA3OC41Mzk4IDk1LjY0NiA3OC4zOTAyQzk1Ljc0MTIgNzguMjM3OCA5NS44MTIgNzguMDQzMyA5NS44NTgyIDc3LjgwNjZDOTUuOTA0NSA3Ny41NjcyIDk1LjkyNzYgNzcuMjgxNSA5NS45Mjc2IDc2Ljk0OTZaIiBmaWxsPSJibGFjayIgZmlsbC1vcGFjaXR5PSIwLjc2Ii8+CjxyZWN0IHdpZHRoPSIxNSIgaGVpZ2h0PSI4LjM1ODIxIiB0cmFuc2Zvcm09InRyYW5zbGF0ZSg4Mi43NjEyIDIyLjUzNzQpIiBmaWxsPSJ3aGl0ZSIvPgo8cGF0aCBkPSJNODUuOTIyOSAyMy4yNDE2VjI5LjIxNjRIODUuMTY3OVYyNC4xODQ0TDgzLjY0NTYgMjQuNzM5NFYyNC4wNTc4TDg1LjgwNDYgMjMuMjQxNkg4NS45MjI5Wk04OS4zNTkzIDI2LjM4ODJMODguNzU1MyAyNi4yMzMxTDg5LjA1MzIgMjMuMjc0M0g5Mi4xMDE4VjIzLjk3MjFIODkuNjkzOUw4OS41MTQzIDI1LjU4ODNDODkuNjIzMiAyNS41MjU3IDg5Ljc2MDYgMjUuNDY3MiA4OS45MjY1IDI1LjQxMjhDOTAuMDk1MiAyNS4zNTg0IDkwLjI4ODQgMjUuMzMxMiA5MC41MDYxIDI1LjMzMTJDOTAuNzgwOSAyNS4zMzEyIDkxLjAyNzEgMjUuMzc4OCA5MS4yNDQ4IDI1LjQ3NEM5MS40NjI0IDI1LjU2NjUgOTEuNjQ3NCAyNS42OTk4IDkxLjc5OTggMjUuODc0QzkxLjk1NDkgMjYuMDQ4MSA5Mi4wNzMyIDI2LjI1NzYgOTIuMTU0OSAyNi41MDI1QzkyLjIzNjUgMjYuNzQ3MyA5Mi4yNzczIDI3LjAyMDggOTIuMjc3MyAyNy4zMjI4QzkyLjI3NzMgMjcuNjA4NSA5Mi4yMzc4IDI3Ljg3MSA5Mi4xNTg5IDI4LjExMDRDOTIuMDgyOCAyOC4zNDk5IDkxLjk2NzEgMjguNTU5NCA5MS44MTIgMjguNzM4OUM5MS42NTcgMjguOTE1OCA5MS40NjExIDI5LjA1MzIgOTEuMjI0NCAyOS4xNTExQzkwLjk5MDQgMjkuMjQ5MSA5MC43MTQyIDI5LjI5ODEgOTAuMzk1OSAyOS4yOTgxQzkwLjE1NjUgMjkuMjk4MSA4OS45MjkzIDI5LjI2NTQgODkuNzE0MyAyOS4yMDAxQzg5LjUwMjEgMjkuMTMyMSA4OS4zMTE3IDI5LjAzMDEgODkuMTQzIDI4Ljg5NEM4OC45NzcgMjguNzU1MyA4OC44NDEgMjguNTgzOSA4OC43MzQ4IDI4LjM3OThDODguNjMxNSAyOC4xNzMgODguNTY2MiAyNy45MzA5IDg4LjUzOSAyNy42NTMzSDg5LjI1NzJDODkuMjg5OSAyNy44NzY1IDg5LjM1NTIgMjguMDY0MiA4OS40NTMxIDI4LjIxNjVDODkuNTUxMSAyOC4zNjg5IDg5LjY3OSAyOC40ODQ1IDg5LjgzNjggMjguNTYzNEM4OS45OTczIDI4LjYzOTYgOTAuMTgzNyAyOC42Nzc3IDkwLjM5NTkgMjguNjc3N0M5MC41NzU0IDI4LjY3NzcgOTAuNzM0NiAyOC42NDY0IDkwLjg3MzQgMjguNTgzOUM5MS4wMTIxIDI4LjUyMTMgOTEuMTI5MSAyOC40MzE1IDkxLjIyNDQgMjguMzE0NUM5MS4zMTk2IDI4LjE5NzUgOTEuMzkxNyAyOC4wNTYgOTEuNDQwNyAyNy44OTAxQzkxLjQ5MjMgMjcuNzI0MSA5MS41MTgyIDI3LjUzNzcgOTEuNTE4MiAyNy4zMzA5QzkxLjUxODIgMjcuMTQzMiA5MS40OTIzIDI2Ljk2OTEgOTEuNDQwNyAyNi44MDg1QzkxLjM4OSAyNi42NDggOTEuMzExNCAyNi41MDc5IDkxLjIwOCAyNi4zODgyQzkxLjEwNzQgMjYuMjY4NSA5MC45ODM2IDI2LjE3NiA5MC44MzY2IDI2LjExMDdDOTAuNjg5NyAyNi4wNDI3IDkwLjUyMSAyNi4wMDg2IDkwLjMzMDYgMjYuMDA4NkM5MC4wNzc1IDI2LjAwODYgODkuODg1NyAyNi4wNDI3IDg5Ljc1NTEgMjYuMTEwN0M4OS42MjczIDI2LjE3ODcgODkuNDk1MyAyNi4yNzEyIDg5LjM1OTMgMjYuMzg4MlpNOTcuMDk3MSAyNS43NzE5VjI2LjY3OEM5Ny4wOTcxIDI3LjE2NSA5Ny4wNTM2IDI3LjU3NTggOTYuOTY2NSAyNy45MTA1Qzk2Ljg3OTUgMjguMjQ1MSA5Ni43NTQzIDI4LjUxNDUgOTYuNTkxMSAyOC43MTg1Qzk2LjQyNzggMjguOTIyNiA5Ni4yMzA2IDI5LjA3MDkgOTUuOTk5MyAyOS4xNjM0Qzk1Ljc3MDggMjkuMjUzMiA5NS41MTIzIDI5LjI5ODEgOTUuMjIzOSAyOS4yOTgxQzk0Ljk5NTMgMjkuMjk4MSA5NC43ODQ1IDI5LjI2OTUgOTQuNTkxMyAyOS4yMTIzQzk0LjM5ODEgMjkuMTU1MiA5NC4yMjQgMjkuMDY0MSA5NC4wNjg5IDI4LjkzODlDOTMuOTE2NiAyOC44MTEgOTMuNzg2IDI4LjY0NTEgOTMuNjc3MSAyOC40NDFDOTMuNTY4MyAyOC4yMzcgOTMuNDg1MyAyNy45ODk0IDkzLjQyODIgMjcuNjk4MkM5My4zNzEgMjcuNDA3MSA5My4zNDI1IDI3LjA2NyA5My4zNDI1IDI2LjY3OFYyNS43NzE5QzkzLjM0MjUgMjUuMjg0OSA5My4zODYgMjQuODc2OCA5My40NzMxIDI0LjU0NzZDOTMuNTYyOSAyNC4yMTg0IDkzLjY4OTQgMjMuOTU0NSA5My44NTI2IDIzLjc1NThDOTQuMDE1OSAyMy41NTQ1IDk0LjIxMTggMjMuNDEwMyA5NC40NDAzIDIzLjMyMzJDOTQuNjcxNiAyMy4yMzYyIDk0LjkzIDIzLjE5MjYgOTUuMjE1NyAyMy4xOTI2Qzk1LjQ0NyAyMy4xOTI2IDk1LjY1OTIgMjMuMjIxMiA5NS44NTI0IDIzLjI3ODNDOTYuMDQ4MyAyMy4zMzI4IDk2LjIyMjQgMjMuNDIxMiA5Ni4zNzQ4IDIzLjU0MzZDOTYuNTI3MSAyMy42NjMzIDk2LjY1NjQgMjMuODIzOSA5Ni43NjI1IDI0LjAyNTJDOTYuODcxMyAyNC4yMjM4IDk2Ljk1NDMgMjQuNDY3MyA5Ny4wMTE0IDI0Ljc1NTdDOTcuMDY4NiAyNS4wNDQxIDk3LjA5NzEgMjUuMzgyOSA5Ny4wOTcxIDI1Ljc3MTlaTTk2LjMzOCAyNi44MDA0VjI1LjY0NTRDOTYuMzM4IDI1LjM3ODggOTYuMzIxNyAyNS4xNDQ4IDk2LjI4OTEgMjQuOTQzNUM5Ni4yNTkxIDI0LjczOTQgOTYuMjE0MiAyNC41NjUzIDk2LjE1NDQgMjQuNDIxMUM5Ni4wOTQ1IDI0LjI3NjkgOTYuMDE4NCAyNC4xNTk5IDk1LjkyNTggMjQuMDcwMUM5NS44MzYxIDIzLjk4MDMgOTUuNzMxMyAyMy45MTUgOTUuNjExNiAyMy44NzQyQzk1LjQ5NDYgMjMuODMwNyA5NS4zNjI2IDIzLjgwODkgOTUuMjE1NyAyMy44MDg5Qzk1LjAzNjIgMjMuODA4OSA5NC44NzcgMjMuODQyOSA5NC43MzgyIDIzLjkxMDlDOTQuNTk5NSAyMy45NzYyIDk0LjQ4MjUgMjQuMDgxIDk0LjM4NzIgMjQuMjI1MkM5NC4yOTQ3IDI0LjM2OTQgOTQuMjI0IDI0LjU1ODUgOTQuMTc1IDI0Ljc5MjVDOTQuMTI2MSAyNS4wMjY0IDk0LjEwMTYgMjUuMzEwOCA5NC4xMDE2IDI1LjY0NTRWMjYuODAwNEM5NC4xMDE2IDI3LjA2NyA5NC4xMTY1IDI3LjMwMjQgOTQuMTQ2NSAyNy41MDY0Qzk0LjE3OTEgMjcuNzEwNSA5NC4yMjY3IDI3Ljg4NzMgOTQuMjg5MyAyOC4wMzdDOTQuMzUxOSAyOC4xODM5IDk0LjQyODEgMjguMzA1IDk0LjUxNzggMjguNDAwMkM5NC42MDc2IDI4LjQ5NTQgOTQuNzExIDI4LjU2NjIgOTQuODI4IDI4LjYxMjRDOTQuOTQ3NyAyOC42NTYgOTUuMDc5NyAyOC42Nzc3IDk1LjIyMzkgMjguNjc3N0M5NS40MDg5IDI4LjY3NzcgOTUuNTcwOCAyOC42NDIzIDk1LjcwOTUgMjguNTcxNkM5NS44NDgzIDI4LjUwMDkgOTUuOTYzOSAyOC4zOTA3IDk2LjA1NjQgMjguMjQxQzk2LjE1MTcgMjguMDg4NyA5Ni4yMjI0IDI3Ljg5NDEgOTYuMjY4NyAyNy42NTc0Qzk2LjMxNDkgMjcuNDE4IDk2LjMzOCAyNy4xMzIzIDk2LjMzOCAyNi44MDA0WiIgZmlsbD0iYmxhY2siIGZpbGwtb3BhY2l0eT0iMC43NiIvPgo8cmVjdCB3aWR0aD0iMTUiIGhlaWdodD0iOC4zNTgyMSIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoODIuNzYxMiA1LjgyMDkyKSIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTg3LjMzNSAxMS44Nzk3VjEyLjVIODMuNDQ1N1YxMS45NTcyTDg1LjM5MjQgOS43OTAxMUM4NS42MzE4IDkuNTIzNDggODUuODE2OCA5LjI5NzY1IDg1Ljk0NzQgOS4xMTI2NEM4Ni4wODA3IDguOTI0OTEgODYuMTczMiA4Ljc1NzU4IDg2LjIyNDkgOC42MTA2NkM4Ni4yNzk0IDguNDYxMDIgODYuMzA2NiA4LjMwODY1IDg2LjMwNjYgOC4xNTM1N0M4Ni4zMDY2IDcuOTU3NjcgODYuMjY1NyA3Ljc4MDgyIDg2LjE4NDEgNy42MjMwMkM4Ni4xMDUyIDcuNDYyNDkgODUuOTg4MiA3LjMzNDYyIDg1LjgzMzEgNy4yMzkzOUM4NS42NzgxIDcuMTQ0MTYgODUuNDkwMyA3LjA5NjU1IDg1LjI2OTkgNy4wOTY1NUM4NS4wMDYgNy4wOTY1NSA4NC43ODU2IDcuMTQ4MjQgODQuNjA4OCA3LjI1MTYzQzg0LjQzNDcgNy4zNTIzIDg0LjMwNDEgNy40OTM3OCA4NC4yMTcgNy42NzYwN0M4NC4xMjk5IDcuODU4MzYgODQuMDg2NCA4LjA2Nzg2IDg0LjA4NjQgOC4zMDQ1N0g4My4zMzE0QzgzLjMzMTQgNy45Njk5MiA4My40MDQ5IDcuNjYzODMgODMuNTUxOCA3LjM4NjMxQzgzLjY5ODcgNy4xMDg3OSA4My45MTY0IDYuODg4NDEgODQuMjA0OCA2LjcyNTE2Qzg0LjQ5MzIgNi41NTkyIDg0Ljg0ODIgNi40NzYyMSA4NS4yNjk5IDYuNDc2MjFDODUuNjQ1NCA2LjQ3NjIxIDg1Ljk2NjUgNi41NDI4NyA4Ni4yMzMxIDYuNjc2MTlDODYuNDk5NyA2LjgwNjc5IDg2LjcwMzggNi45OTE4IDg2Ljg0NTMgNy4yMzEyM0M4Ni45ODk1IDcuNDY3OTMgODcuMDYxNiA3Ljc0NTQ1IDg3LjA2MTYgOC4wNjM3OEM4Ny4wNjE2IDguMjM3OTEgODcuMDMxNiA4LjQxNDc2IDg2Ljk3MTggOC41OTQzM0M4Ni45MTQ3IDguNzcxMTggODYuODM0NCA4Ljk0ODAzIDg2LjczMSA5LjEyNDg4Qzg2LjYzMDMgOS4zMDE3MyA4Ni41MTIgOS40NzU4NiA4Ni4zNzU5IDkuNjQ3MjdDODYuMjQyNiA5LjgxODY4IDg2LjA5OTggOS45ODczNyA4NS45NDc0IDEwLjE1MzNMODQuMzU1OCAxMS44Nzk3SDg3LjMzNVpNOTIuMTM0NSA5LjA1NTVWOS45NjE1MkM5Mi4xMzQ1IDEwLjQ0ODUgOTIuMDkwOSAxMC44NTk0IDkyLjAwMzkgMTEuMTk0QzkxLjkxNjggMTEuNTI4NyA5MS43OTE2IDExLjc5OCA5MS42Mjg0IDEyLjAwMjFDOTEuNDY1MSAxMi4yMDYyIDkxLjI2NzkgMTIuMzU0NCA5MS4wMzY2IDEyLjQ0NjlDOTAuODA4MSAxMi41MzY3IDkwLjU0OTYgMTIuNTgxNiA5MC4yNjEyIDEyLjU4MTZDOTAuMDMyNyAxMi41ODE2IDg5LjgyMTggMTIuNTUzMSA4OS42Mjg2IDEyLjQ5NTlDODkuNDM1NCAxMi40Mzg4IDg5LjI2MTMgMTIuMzQ3NiA4OS4xMDYyIDEyLjIyMjVDODguOTUzOSAxMi4wOTQ2IDg4LjgyMzMgMTEuOTI4NiA4OC43MTQ0IDExLjcyNDZDODguNjA1NiAxMS41MjA1IDg4LjUyMjYgMTEuMjcyOSA4OC40NjU1IDEwLjk4MThDODguNDA4NCAxMC42OTA3IDg4LjM3OTggMTAuMzUwNiA4OC4zNzk4IDkuOTYxNTJWOS4wNTU1Qzg4LjM3OTggOC41Njg0OSA4OC40MjMzIDguMTYwMzcgODguNTEwNCA3LjgzMTE2Qzg4LjYwMDIgNy41MDE5NCA4OC43MjY3IDcuMjM4MDMgODguODg5OSA3LjAzOTQxQzg5LjA1MzIgNi44MzgwOCA4OS4yNDkxIDYuNjkzODcgODkuNDc3NiA2LjYwNjgxQzg5LjcwODkgNi41MTk3NSA4OS45Njc0IDYuNDc2MjEgOTAuMjUzIDYuNDc2MjFDOTAuNDg0MyA2LjQ3NjIxIDkwLjY5NjUgNi41MDQ3OCA5MC44ODk3IDYuNTYxOTJDOTEuMDg1NiA2LjYxNjMzIDkxLjI1OTcgNi43MDQ3NiA5MS40MTIxIDYuODI3MTlDOTEuNTY0NCA2Ljk0NjkxIDkxLjY5MzcgNy4xMDc0MyA5MS43OTk4IDcuMzA4NzdDOTEuOTA4NiA3LjUwNzM5IDkxLjk5MTYgNy43NTA4OSA5Mi4wNDg3IDguMDM5M0M5Mi4xMDU5IDguMzI3NyA5Mi4xMzQ1IDguNjY2NDMgOTIuMTM0NSA5LjA1NTVaTTkxLjM3NTQgMTAuMDg0VjguOTI4OTlDOTEuMzc1NCA4LjY2MjM1IDkxLjM1OSA4LjQyODM3IDkxLjMyNjQgOC4yMjcwM0M5MS4yOTY1IDguMDIyOTcgOTEuMjUxNiA3Ljg0ODg0IDkxLjE5MTcgNy43MDQ2NEM5MS4xMzE4IDcuNTYwNDQgOTEuMDU1NyA3LjQ0MzQ1IDkwLjk2MzIgNy4zNTM2NkM5MC44NzM0IDcuMjYzODggOTAuNzY4NiA3LjE5ODU4IDkwLjY0ODkgNy4xNTc3N0M5MC41MzE5IDcuMTE0MjMgOTAuNCA3LjA5MjQ3IDkwLjI1MyA3LjA5MjQ3QzkwLjA3MzUgNy4wOTI0NyA4OS45MTQzIDcuMTI2NDggODkuNzc1NSA3LjE5NDVDODkuNjM2OCA3LjI1OTc5IDg5LjUxOTggNy4zNjQ1NCA4OS40MjQ2IDcuNTA4NzVDODkuMzMyMSA3LjY1Mjk1IDg5LjI2MTMgNy44NDIwNCA4OS4yMTIzIDguMDc2MDNDODkuMTYzNCA4LjMxMDAxIDg5LjEzODkgOC41OTQzMyA4OS4xMzg5IDguOTI4OTlWMTAuMDg0Qzg5LjEzODkgMTAuMzUwNiA4OS4xNTM4IDEwLjU4NTkgODkuMTgzOCAxMC43OUM4OS4yMTY0IDEwLjk5NDEgODkuMjY0IDExLjE3MDkgODkuMzI2NiAxMS4zMjA1Qzg5LjM4OTIgMTEuNDY3NSA4OS40NjU0IDExLjU4ODUgODkuNTU1MiAxMS42ODM4Qzg5LjY0NDkgMTEuNzc5IDg5Ljc0ODMgMTEuODQ5NyA4OS44NjUzIDExLjg5NkM4OS45ODUgMTEuOTM5NSA5MC4xMTcgMTEuOTYxMyA5MC4yNjEyIDExLjk2MTNDOTAuNDQ2MiAxMS45NjEzIDkwLjYwODEgMTEuOTI1OSA5MC43NDY5IDExLjg1NTJDOTAuODg1NiAxMS43ODQ0IDkxLjAwMTIgMTEuNjc0MiA5MS4wOTM4IDExLjUyNDZDOTEuMTg5IDExLjM3MjIgOTEuMjU5NyAxMS4xNzc3IDkxLjMwNiAxMC45NDFDOTEuMzUyMiAxMC43MDE2IDkxLjM3NTQgMTAuNDE1OSA5MS4zNzU0IDEwLjA4NFpNOTcuMDk3MSA5LjA1NTVWOS45NjE1MkM5Ny4wOTcxIDEwLjQ0ODUgOTcuMDUzNiAxMC44NTk0IDk2Ljk2NjUgMTEuMTk0Qzk2Ljg3OTUgMTEuNTI4NyA5Ni43NTQzIDExLjc5OCA5Ni41OTExIDEyLjAwMjFDOTYuNDI3OCAxMi4yMDYyIDk2LjIzMDYgMTIuMzU0NCA5NS45OTkzIDEyLjQ0NjlDOTUuNzcwOCAxMi41MzY3IDk1LjUxMjMgMTIuNTgxNiA5NS4yMjM5IDEyLjU4MTZDOTQuOTk1MyAxMi41ODE2IDk0Ljc4NDUgMTIuNTUzMSA5NC41OTEzIDEyLjQ5NTlDOTQuMzk4MSAxMi40Mzg4IDk0LjIyNCAxMi4zNDc2IDk0LjA2ODkgMTIuMjIyNUM5My45MTY2IDEyLjA5NDYgOTMuNzg2IDExLjkyODYgOTMuNjc3MSAxMS43MjQ2QzkzLjU2ODMgMTEuNTIwNSA5My40ODUzIDExLjI3MjkgOTMuNDI4MiAxMC45ODE4QzkzLjM3MSAxMC42OTA3IDkzLjM0MjUgMTAuMzUwNiA5My4zNDI1IDkuOTYxNTJWOS4wNTU1QzkzLjM0MjUgOC41Njg0OSA5My4zODYgOC4xNjAzNyA5My40NzMxIDcuODMxMTZDOTMuNTYyOSA3LjUwMTk0IDkzLjY4OTQgNy4yMzgwMyA5My44NTI2IDcuMDM5NDFDOTQuMDE1OSA2LjgzODA4IDk0LjIxMTggNi42OTM4NyA5NC40NDAzIDYuNjA2ODFDOTQuNjcxNiA2LjUxOTc1IDk0LjkzIDYuNDc2MjEgOTUuMjE1NyA2LjQ3NjIxQzk1LjQ0NyA2LjQ3NjIxIDk1LjY1OTIgNi41MDQ3OCA5NS44NTI0IDYuNTYxOTJDOTYuMDQ4MyA2LjYxNjMzIDk2LjIyMjQgNi43MDQ3NiA5Ni4zNzQ4IDYuODI3MTlDOTYuNTI3MSA2Ljk0NjkxIDk2LjY1NjQgNy4xMDc0MyA5Ni43NjI1IDcuMzA4NzdDOTYuODcxMyA3LjUwNzM5IDk2Ljk1NDMgNy43NTA4OSA5Ny4wMTE0IDguMDM5M0M5Ny4wNjg2IDguMzI3NyA5Ny4wOTcxIDguNjY2NDMgOTcuMDk3MSA5LjA1NTVaTTk2LjMzOCAxMC4wODRWOC45Mjg5OUM5Ni4zMzggOC42NjIzNSA5Ni4zMjE3IDguNDI4MzcgOTYuMjg5MSA4LjIyNzAzQzk2LjI1OTEgOC4wMjI5NyA5Ni4yMTQyIDcuODQ4ODQgOTYuMTU0NCA3LjcwNDY0Qzk2LjA5NDUgNy41NjA0NCA5Ni4wMTg0IDcuNDQzNDUgOTUuOTI1OCA3LjM1MzY2Qzk1LjgzNjEgNy4yNjM4OCA5NS43MzEzIDcuMTk4NTggOTUuNjExNiA3LjE1Nzc3Qzk1LjQ5NDYgNy4xMTQyMyA5NS4zNjI2IDcuMDkyNDcgOTUuMjE1NyA3LjA5MjQ3Qzk1LjAzNjIgNy4wOTI0NyA5NC44NzcgNy4xMjY0OCA5NC43MzgyIDcuMTk0NUM5NC41OTk1IDcuMjU5NzkgOTQuNDgyNSA3LjM2NDU0IDk0LjM4NzIgNy41MDg3NUM5NC4yOTQ3IDcuNjUyOTUgOTQuMjI0IDcuODQyMDQgOTQuMTc1IDguMDc2MDNDOTQuMTI2MSA4LjMxMDAxIDk0LjEwMTYgOC41OTQzMyA5NC4xMDE2IDguOTI4OTlWMTAuMDg0Qzk0LjEwMTYgMTAuMzUwNiA5NC4xMTY1IDEwLjU4NTkgOTQuMTQ2NSAxMC43OUM5NC4xNzkxIDEwLjk5NDEgOTQuMjI2NyAxMS4xNzA5IDk0LjI4OTMgMTEuMzIwNUM5NC4zNTE5IDExLjQ2NzUgOTQuNDI4MSAxMS41ODg1IDk0LjUxNzggMTEuNjgzOEM5NC42MDc2IDExLjc3OSA5NC43MTEgMTEuODQ5NyA5NC44MjggMTEuODk2Qzk0Ljk0NzcgMTEuOTM5NSA5NS4wNzk3IDExLjk2MTMgOTUuMjIzOSAxMS45NjEzQzk1LjQwODkgMTEuOTYxMyA5NS41NzA4IDExLjkyNTkgOTUuNzA5NSAxMS44NTUyQzk1Ljg0ODMgMTEuNzg0NCA5NS45NjM5IDExLjY3NDIgOTYuMDU2NCAxMS41MjQ2Qzk2LjE1MTcgMTEuMzcyMiA5Ni4yMjI0IDExLjE3NzcgOTYuMjY4NyAxMC45NDFDOTYuMzE0OSAxMC43MDE2IDk2LjMzOCAxMC40MTU5IDk2LjMzOCAxMC4wODRaIiBmaWxsPSJibGFjayIgZmlsbC1vcGFjaXR5PSIwLjc2Ii8+CjxwYXRoIGQ9Ik03Ni4yNzY1IDkyLjI2NEM3OC41NjEgOTYuNDIwNSA4MS45MTkgOTkuODg3NiA4Ni4wMDAzIDEwMi4zMDRDOTAuMDgxNiAxMDQuNzIgOTQuNzM2NSAxMDUuOTk2IDk5LjQ3OTQgMTA2QzEwNC4yMjIgMTA2LjAwNCAxMDguODc5IDEwNC43MzQgMTEyLjk2NCAxMDIuMzI1QzExNy4wNDkgOTkuOTE0OSAxMjAuNDEzIDk2LjQ1MyAxMjIuNzA0IDkyLjMwMDJMOTkuNSA3OS41TDc2LjI3NjUgOTIuMjY0WiIgZmlsbD0iIzRCOTNGRiIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjA0NDc4Ii8+CjxwYXRoIGQ9Ik0xMzUuMDExIDk5LjM0NDZDMTM4LjM3NyA5My4yNTM3IDE0MC4wOTUgODYuMzkwMyAxMzkuOTk2IDc5LjQzMjNDMTM5Ljg5NyA3Mi40NzQyIDEzNy45ODUgNjUuNjYyMyAxMzQuNDQ4IDU5LjY2OTVDMTMwLjkxMSA1My42NzY2IDEyNS44NzIgNDguNzEwMSAxMTkuODI5IDQ1LjI2MDZDMTEzLjc4NSA0MS44MTEgMTA2Ljk0NiAzOS45OTc4IDk5Ljk4NzMgNDBMMTAwIDgwTDEzNS4wMTEgOTkuMzQ0NloiIGZpbGw9IiNGRjRENUEiIHN0cm9rZT0id2hpdGUiIHN0cm9rZS13aWR0aD0iMS4wNDQ3OCIvPgo8ZyBmaWx0ZXI9InVybCgjZmlsdGVyMF9kaV80NjE3XzQzNDk5KSI+CjxwYXRoIGQ9Ik05OS40Nzc2IDI3Ljc2MTJDOTAuMzk4MSAyNy43NjEyIDgxLjQ3NTMgMzAuMTI3NyA3My41ODkxIDM0LjYyNzRDNjUuNzAzIDM5LjEyNyA1OS4xMjU4IDQ1LjYwNDQgNTQuNTA2MSA1My40MjA4QzQ5Ljg4NjQgNjEuMjM3MyA0Ny4zODM3IDcwLjEyMjggNDcuMjQ0OSA3OS4yMDEzQzQ3LjEwNjEgODguMjc5OCA0OS4zMzU5IDk3LjIzNzcgNTMuNzE0NCAxMDUuMTkyTDk5LjQ3NzYgODBWMjcuNzYxMloiIGZpbGw9IiMwODg3MkIiLz4KPHBhdGggZD0iTTk5LjQ3NzYgMjcuNzYxMkM5MC4zOTgxIDI3Ljc2MTIgODEuNDc1MyAzMC4xMjc3IDczLjU4OTEgMzQuNjI3NEM2NS43MDMgMzkuMTI3IDU5LjEyNTggNDUuNjA0NCA1NC41MDYxIDUzLjQyMDhDNDkuODg2NCA2MS4yMzczIDQ3LjM4MzcgNzAuMTIyOCA0Ny4yNDQ5IDc5LjIwMTNDNDcuMTA2MSA4OC4yNzk4IDQ5LjMzNTkgOTcuMjM3NyA1My43MTQ0IDEwNS4xOTJMOTkuNDc3NiA4MFYyNy43NjEyWiIgc3Ryb2tlPSJ3aGl0ZSIgc3Ryb2tlLXdpZHRoPSIxLjA0NDc4Ii8+CjwvZz4KPHBhdGggZD0iTTc1LjEzODggNjcuOTU5OEw3My40NDUxIDczSDcyLjI1NzVMNzQuNDc3NiA2Ny4wNTc4SDc1LjIzNjdMNzUuMTM4OCA2Ny45NTk4Wk03Ni41NTQ5IDczTDc0Ljg1MzEgNjcuOTU5OEw3NC43NTEgNjcuMDU3OEg3NS41MTQyTDc3Ljc0NjYgNzNINzYuNTU0OVpNNzYuNDc3NCA3MC43OTIxVjcxLjY3NzdINzMuMjlWNzAuNzkyMUg3Ni40Nzc0WiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTEwMC4zMSA5NC4wNDUxSDEwMS40MzJDMTAxLjQwMiA5NC40NDI0IDEwMS4yOTIgOTQuNzk0NyAxMDEuMTAyIDk1LjEwMjFDMTAwLjkxMSA5NS40MDY5IDEwMC42NDYgOTUuNjQ2MyAxMDAuMzA2IDk1LjgyMDRDOTkuOTY1OSA5NS45OTQ2IDk5LjU1MzcgOTYuMDgxNiA5OS4wNjk0IDk2LjA4MTZDOTguNjk2NiA5Ni4wODE2IDk4LjM2MDYgOTYuMDE2MyA5OC4wNjEzIDk1Ljg4NTdDOTcuNzY0OCA5NS43NTI0IDk3LjUxMDQgOTUuNTYzMyA5Ny4yOTgyIDk1LjMxODRDOTcuMDg4NyA5NS4wNzA5IDk2LjkyODEgOTQuNzc0MyA5Ni44MTY2IDk0LjQyODhDOTYuNzA1IDk0LjA4MDUgOTYuNjQ5MyA5My42OTAxIDk2LjY0OTMgOTMuMjU3NVY5Mi44MDQ1Qzk2LjY0OTMgOTIuMzcxOSA5Ni43MDY0IDkxLjk4MTQgOTYuODIwNyA5MS42MzMyQzk2LjkzNDkgOTEuMjg0OSA5Ny4wOTgyIDkwLjk4ODMgOTcuMzEwNCA5MC43NDM1Qzk3LjUyNTMgOTAuNDk1OSA5Ny43ODI1IDkwLjMwNTQgOTguMDgxNyA5MC4xNzIxQzk4LjM4MzcgOTAuMDM4OCA5OC43MjExIDg5Ljk3MjEgOTkuMDkzOSA4OS45NzIxQzk5LjU3ODIgODkuOTcyMSA5OS45ODc2IDkwLjA2MTkgMTAwLjMyMiA5MC4yNDE1QzEwMC42NTcgOTAuNDE4MyAxMDAuOTE3IDkwLjY2MTggMTAxLjEwMiA5MC45NzJDMTAxLjI4NyA5MS4yODIyIDEwMS4zOTggOTEuNjM4NiAxMDEuNDM2IDkyLjA0MTNIMTAwLjMxNEMxMDAuMjkyIDkxLjc5MSAxMDAuMjM4IDkxLjU3ODcgMTAwLjE1MSA5MS40MDQ2QzEwMC4wNjcgOTEuMjMwNSA5OS45Mzg3IDkxLjA5ODUgOTkuNzY3MyA5MS4wMDg3Qzk5LjU5ODYgOTAuOTE2MiA5OS4zNzQxIDkwLjg3IDk5LjA5MzkgOTAuODdDOTguODc2MiA5MC44NyA5OC42ODQ0IDkwLjkxMDggOTguNTE4NCA5MC45OTI0Qzk4LjM1NTIgOTEuMDc0IDk4LjIxOTEgOTEuMTk2NSA5OC4xMTAzIDkxLjM1OTdDOTguMDAxNSA5MS41MjAzIDk3LjkxOTkgOTEuNzIxNiA5Ny44NjU0IDkxLjk2MzdDOTcuODExIDkyLjIwMzIgOTcuNzgzOCA5Mi40ODA3IDk3Ljc4MzggOTIuNzk2M1Y5My4yNTc1Qzk3Ljc4MzggOTMuNTU5NSA5Ny44MDgzIDkzLjgzMDIgOTcuODU3MyA5NC4wNjk2Qzk3LjkwNjIgOTQuMzA5IDk3Ljk4MjQgOTQuNTExNyA5OC4wODU4IDk0LjY3NzdDOTguMTg5MiA5NC44NDM3IDk4LjMyMjUgOTQuOTcwMiA5OC40ODU4IDk1LjA1NzNDOTguNjQ5IDk1LjE0NDMgOTguODQzNiA5NS4xODc4IDk5LjA2OTQgOTUuMTg3OEM5OS4zNDQyIDk1LjE4NzggOTkuNTY3MyA5NS4xNDQzIDk5LjczODcgOTUuMDU3M0M5OS45MTI4IDk0Ljk3MDIgMTAwLjA0NSA5NC44NDIzIDEwMC4xMzUgOTQuNjczNkMxMDAuMjI3IDk0LjUwNDkgMTAwLjI4NiA5NC4yOTU0IDEwMC4zMSA5NC4wNDUxWiIgZmlsbD0id2hpdGUiLz4KPHBhdGggZD0iTTExNy4xOTIgNzAuMzUxM0gxMTUuNjc0TDExNS42NjUgNjkuNTU5NkgxMTYuOTY3QzExNy4xOSA2OS41NTk2IDExNy4zNzcgNjkuNTI4MyAxMTcuNTI2IDY5LjQ2NTdDMTE3LjY3NiA2OS40MDMxIDExNy43ODkgNjkuMzEyIDExNy44NjUgNjkuMTkyM0MxMTcuOTQ0IDY5LjA3MjYgMTE3Ljk4NCA2OC45MjcgMTE3Ljk4NCA2OC43NTU2QzExNy45ODQgNjguNTY1MSAxMTcuOTQ3IDY4LjQxMDEgMTE3Ljg3MyA2OC4yOTAzQzExNy44MDMgNjguMTcwNiAxMTcuNjkxIDY4LjA4MzYgMTE3LjUzOSA2OC4wMjkyQzExNy4zODYgNjcuOTcyIDExNy4xOTMgNjcuOTQzNCAxMTYuOTU5IDY3Ljk0MzRIMTE2LjAzN1Y3M0gxMTQuOTE1VjY3LjA1NzhIMTE2Ljk1OUMxMTcuMjk3IDY3LjA1NzggMTE3LjU5NyA2Ny4wOTA1IDExNy44NjEgNjcuMTU1OEMxMTguMTI4IDY3LjIxODQgMTE4LjM1NCA2Ny4zMTYzIDExOC41MzkgNjcuNDQ5NkMxMTguNzI0IDY3LjU4MjkgMTE4Ljg2NCA2Ny43NTAzIDExOC45NTkgNjcuOTUxNkMxMTkuMDU3IDY4LjE1MjkgMTE5LjEwNiA2OC4zOTI0IDExOS4xMDYgNjguNjY5OUMxMTkuMTA2IDY4LjkxNDggMTE5LjA0OSA2OS4xNDA2IDExOC45MzQgNjkuMzQ3NEMxMTguODIzIDY5LjU1MTQgMTE4LjY1IDY5LjcxNzQgMTE4LjQxNiA2OS44NDUzQzExOC4xODUgNjkuOTczMSAxMTcuODkxIDcwLjA0NjYgMTE3LjUzNSA3MC4wNjU2TDExNy4xOTIgNzAuMzUxM1pNMTE3LjE0MyA3M0gxMTUuMzQzTDExNS44MTIgNzIuMTE4NUgxMTcuMTQzQzExNy4zNjYgNzIuMTE4NSAxMTcuNTUgNzIuMDgxNyAxMTcuNjk0IDcyLjAwODNDMTE3Ljg0MSA3MS45MzQ4IDExNy45NSA3MS44MzQxIDExOC4wMiA3MS43MDYzQzExOC4wOTQgNzEuNTc1NyAxMTguMTMgNzEuNDI2IDExOC4xMyA3MS4yNTczQzExOC4xMyA3MS4wNzIzIDExOC4wOTggNzAuOTExOCAxMTguMDMzIDcwLjc3NThDMTE3Ljk3IDcwLjYzOTcgMTE3Ljg2OSA3MC41MzUgMTE3LjczMSA3MC40NjE1QzExNy41OTQgNzAuMzg4MSAxMTcuNDE1IDcwLjM1MTMgMTE3LjE5MiA3MC4zNTEzSDExNi4wMjVMMTE2LjAzMyA2OS41NTk2SDExNy41MThMMTE3Ljc3NSA2OS44NjU3QzExOC4xMTggNjkuODY4NCAxMTguMzk3IDY5LjkzNjQgMTE4LjYxMiA3MC4wNjk3QzExOC44MyA3MC4yMDMgMTE4Ljk5IDcwLjM3NDUgMTE5LjA5NCA3MC41ODRDMTE5LjE5NyA3MC43OTM1IDExOS4yNDkgNzEuMDE5MyAxMTkuMjQ5IDcxLjI2MTRDMTE5LjI0OSA3MS42NDIzIDExOS4xNjYgNzEuOTYyIDExOSA3Mi4yMjA1QzExOC44MzcgNzIuNDc5IDExOC41OTcgNzIuNjczNSAxMTguMjgxIDcyLjgwNDFDMTE3Ljk2OSA3Mi45MzQ3IDExNy41ODkgNzMgMTE3LjE0MyA3M1oiIGZpbGw9IndoaXRlIi8+CjxkZWZzPgo8ZmlsdGVyIGlkPSJmaWx0ZXIwX2RpXzQ2MTdfNDM0OTkiIHg9IjM0LjE3OTEiIHk9IjE0LjcwMTYiIHdpZHRoPSI3OC4zNTgyIiBoZWlnaHQ9IjEwMy43MzciIGZpbHRlclVuaXRzPSJ1c2VyU3BhY2VPblVzZSIgY29sb3ItaW50ZXJwb2xhdGlvbi1maWx0ZXJzPSJzUkdCIj4KPGZlRmxvb2QgZmxvb2Qtb3BhY2l0eT0iMCIgcmVzdWx0PSJCYWNrZ3JvdW5kSW1hZ2VGaXgiLz4KPGZlQ29sb3JNYXRyaXggaW49IlNvdXJjZUFscGhhIiB0eXBlPSJtYXRyaXgiIHZhbHVlcz0iMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMTI3IDAiIHJlc3VsdD0iaGFyZEFscGhhIi8+CjxmZU9mZnNldC8+CjxmZUdhdXNzaWFuQmx1ciBzdGREZXZpYXRpb249IjYuMjY4NjYiLz4KPGZlQ29tcG9zaXRlIGluMj0iaGFyZEFscGhhIiBvcGVyYXRvcj0ib3V0Ii8+CjxmZUNvbG9yTWF0cml4IHR5cGU9Im1hdHJpeCIgdmFsdWVzPSIwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwLjM2IDAiLz4KPGZlQmxlbmQgbW9kZT0ibm9ybWFsIiBpbjI9IkJhY2tncm91bmRJbWFnZUZpeCIgcmVzdWx0PSJlZmZlY3QxX2Ryb3BTaGFkb3dfNDYxN180MzQ5OSIvPgo8ZmVCbGVuZCBtb2RlPSJub3JtYWwiIGluPSJTb3VyY2VHcmFwaGljIiBpbjI9ImVmZmVjdDFfZHJvcFNoYWRvd180NjE3XzQzNDk5IiByZXN1bHQ9InNoYXBlIi8+CjxmZUNvbG9yTWF0cml4IGluPSJTb3VyY2VBbHBoYSIgdHlwZT0ibWF0cml4IiB2YWx1ZXM9IjAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDEyNyAwIiByZXN1bHQ9ImhhcmRBbHBoYSIvPgo8ZmVPZmZzZXQvPgo8ZmVHYXVzc2lhbkJsdXIgc3RkRGV2aWF0aW9uPSIzLjEzNDMzIi8+CjxmZUNvbXBvc2l0ZSBpbjI9ImhhcmRBbHBoYSIgb3BlcmF0b3I9ImFyaXRobWV0aWMiIGsyPSItMSIgazM9IjEiLz4KPGZlQ29sb3JNYXRyaXggdHlwZT0ibWF0cml4IiB2YWx1ZXM9IjAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAuNDUgMCIvPgo8ZmVCbGVuZCBtb2RlPSJub3JtYWwiIGluMj0ic2hhcGUiIHJlc3VsdD0iZWZmZWN0Ml9pbm5lclNoYWRvd180NjE3XzQzNDk5Ii8+CjwvZmlsdGVyPgo8L2RlZnM+Cjwvc3ZnPgo=", + "description": "Displays the latest values of the attributes or time-series data in a polar area chart. Supports numeric values only.", "descriptor": { "type": "latest", - "sizeX": 7, - "sizeY": 5, - "resources": [ - { - "url": "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.3.0/Chart.min.js" - } - ], - "templateHtml": "\n", + "sizeX": 5, + "sizeY": 4, + "resources": [], + "templateHtml": "\n", "templateCss": "", - "controllerScript": "self.onInit = function() {\n $scope = self.ctx.$scope;\n utils = $scope.$injector.get(self.ctx.servicesMap.get('utils'));\n settings = utils.deepClone(self.ctx.settings) || {};\n settings.showTooltip = utils.defaultValue(settings.showTooltip, true);\n \n Chart.defaults.global.tooltips.enabled = settings.showTooltip;\n \n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n\n pieData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n pieData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n \n var floatingPoint;\n if (typeof self.ctx.decimals !== 'undefined' && self.ctx.decimals !== null) {\n floatingPoint = self.ctx.widget.config.decimals;\n } else {\n floatingPoint = 2;\n }\n\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'polarArea',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scale: {\n ticks: {\n callback: function(tick) {\n \treturn tick.toFixed(floatingPoint);\n }\n }\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n try {\n self.ctx.chart.resize();\n } catch (e) {}\n }\n}\n\nself.onDestroy = function() {\n self.ctx.chart.destroy();\n self.ctx.chart = null;\n}\n", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.polarAreaChartWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.polarAreaChartWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n previewWidth: '500px',\n previewHeight: '380px',\n embedTitlePanel: true,\n defaultDataKeysFunction: function() {\n return [{ name: 'windPower', label: 'Wind', type: 'timeseries', color: '#08872B' },\n { name: 'solarPower', label: 'Solar', type: 'timeseries', color: '#FF4D5A' },\n { name: 'hydroelectricPower', label: 'Hydroelectric', type: 'timeseries', color: '#FFDE30' }];\n }\n };\n};\n\nself.onDestroy = function() {\n};\n\n", "settingsSchema": "", - "dataKeySettingsSchema": "{}\n", - "settingsDirective": "tb-chart-widget-settings", - "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fifth\",\"color\":\"#607d8b\",\"settings\":{},\"_hash\":0.2074391823443591,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Polar Area\"}" + "dataKeySettingsSchema": "", + "settingsDirective": "tb-polar-area-chart-widget-settings", + "hasBasicMode": true, + "basicModeDirective": "tb-polar-area-chart-basic-config", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Wind\",\"color\":\"#08872B\",\"settings\":{},\"_hash\":0.7227918773301678,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Solar\",\"color\":\"#FF4D5A\",\"settings\":{},\"_hash\":0.2392660816082064,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Hydroelectric\",\"color\":\"#FFDE30\",\"settings\":{},\"_hash\":0.7051898468567794,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 200;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 200) {\\n\\tvalue = 200;\\n}\\nreturn value;\",\"decimals\":0,\"aggregationType\":null,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":null,\"padding\":\"0px\",\"settings\":{},\"title\":\"Polar area\",\"dropShadow\":true,\"enableFullscreen\":true,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"units\":null,\"decimals\":0,\"useDashboardTimewindow\":true,\"showLegend\":false,\"widgetStyle\":{},\"actions\":{\"headerButton\":[]},\"configMode\":\"basic\",\"displayTimewindow\":true,\"margin\":\"0px\",\"borderRadius\":\"\",\"widgetCss\":\"\",\"pageSize\":1024,\"noDataDisplayMessage\":\"\",\"showTitleIcon\":false,\"titleTooltip\":\"\",\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleIcon\":\"bar_chart\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"timewindowStyle\":{\"showIcon\":false,\"iconSize\":\"24px\",\"icon\":null,\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"400\",\"style\":\"normal\",\"lineHeight\":\"16px\"},\"color\":\"rgba(0, 0, 0, 0.38)\",\"displayTypePrefix\":true}}" }, - "externalId": null + "tags": [ + "polar", + "polar area", + "bars", + "polar area chart" + ] } \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_types/polar_area_deprecated.json b/application/src/main/data/json/system/widget_types/polar_area_deprecated.json new file mode 100644 index 0000000000..b37392c867 --- /dev/null +++ b/application/src/main/data/json/system/widget_types/polar_area_deprecated.json @@ -0,0 +1,25 @@ +{ + "fqn": "charts.polar_area_chart_js", + "name": "Polar Area", + "deprecated": true, + "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAIAAADGnbT+AAAABmJLR0QA/wD/AP+gvaeTAAAYvUlEQVR42u2deXAU15nA+W+zm73/yV7Zqq09aje7SeXY2kocZ7cSH7GdxCG+cJxybHw7Xt/4WB+AY8CAMeEGc4MNAgcMxhgB5r4RkpAAHYCEDoRu0DX31fuTejyaEWjm9fTr7tejefWKksRMT/f3fvO+733He6O0fMs3C9qovAgytlgs5vf7Ez+3tbX19fXlxZIHK/sWjUYLCgqmTZtWVFSkUzVr1qxVq1ZNnz79yJEjefnkwcqy7d+/f8OGDfCk/1pdXb18+XJ+8Pl8EydOzMsnD1aWberUqZ9//jlTVElJCb8ePHhw27Zt+n+98cYbkUgkL6I8WNm0cePGbd++vbm5+Z133jl//vzu3bt37Nih/9f48eMThle+5cEa1jwPBAI9PT0dHR2XLl1K/P2VV17Rf9i3b19hYeGhQ4e2bt2amLGwwPSfeQtv7O3t5SIJvZkHa4SS5PF4WN/V19djOZWVlVVUVDAnNTQ0tLS0JF6Gqc5f+AFL6+jRoxcuXFiwYAG/guCkSZMSL+MtvIy3cxEuxQX5lYvzESOWs1EjDSb02rlz5xj+qqqqxsbGzs5Or9ebmHuGtKampilTpsydO3fevHnhcJi/rFy5cvbs2ZMnTz59+vRwC0k+hctycT6CD+Lj+NCRBlnug8VIX7lypa6urry8nJFGbaGzDNndKLjkX0OhkDgifBDTGx/KR3MD3AY3MxzHebDcMT91dXWhvBjOmpoaphCAcPaWuAFug5vRCeP2cngOy0GwmGCYIU6dOoUOYiAVdAqgVbkxbg99yq0OmRHzYCnX0HHMByCFbeSK0eImuVVumNvm5vNgKaf1Ll++jBFTWVnJTOA6/cINc9vcPI+ABZYHS4nGSDAkqBVsZLc/C49w9uxZHgfzKw+Wk4qPrzgjkWNKBLzwhPFors6hcCVYmCa1tbV4I3Pgmz1c49HOnDnDqjYYDObBssMcwc2Ntdva2prz/kbcXTwmD8sju+5h3QQWzmvsDxZQVnyJGbjO7p6K2oY9RWUFhXsXrN8yfeVHExeufmnm4t9MnpP8ytf3BaYdDS4qDa6rDBXWhsvaIt0BC0cd7xfzFg/O4+fBkj9RsSzH6yNX93X19p2oOAtGk5aseey3v3vwzXeH68nv+ocFfVf3by/33LnR99KewMpToVPt0Yhs1zprFN3p5ZapywVgkZ2CMcu3Vo/WmWzhSOT0+boPtu5iKkpDklGwhvSvLe4bs9k343iwpDUSlUQCj49liVFPmmEeLLMNBw9GBv+aHpXI8dPV89dteXLSbHGesgYruf/nCs/LewI7LoT9Er4XGvk5CAS/XR6s7NUfCQLYFibz6S61d360c//TU+dnwZMUsBL960s9r+0LFLeYDTEhEFbEFy9eVFktKgoWFisOKtRf1pE+hM4U9daiD8zwJBesRL9tvXf16ZDPxASGWBAOIpJiHowUsPhG4sJhpZ21FXWg5NSrs5ZKQcoKsPT+nRWeecXBHhMrStwQCErNDGnlwMKNjg2RXcgsGo3tKy5/YcYiiUhZB1ZCP04/FuzKFi+WyYhLwXCWWmDpYsoulFF5oWH8/FXSkbIaLL1/c5lnYWkwlJWTAnEhNNWCEAqBpbtqslhLt1/pnr1mk0VI2QOW3m8q8O5tyMamRBsiOqWWiqqAhUMhC3OBVdHeE+WPvz3LUqpsA0vvD33mb/XEsmALAZr3y+QUWHzVEIrR1LzOrp7pK9ZbjZT9YOma8eOzhpd7CBAxKpLR5TxYGAdM40bnqiNlFdm5Ol0Blt6f3uk3GoWELYSpAlsOg6WvAQ3ZVcT8cXjahpSDYNG/v9pT3mbMpEeYWS+AcgQsZimjIrjS00fA2GaqHASL/q/v931UZay+SF8nOhtSdAwsfOsYBIYWMrUXm5+ZNt9+qpwFS+8TDgQiRrQi2hDxOuiXdwYs4i2EI5KL2TM2UhJsWP0pCxb98UJ/wIgvAvEiZKfiic6ARXSZDBDx1xdXnHvkrZlOUaUIWHRScXqDBkAhnkiseqSApZc6iUeXdx0rHTt+hoNUqQOWHsBu9xoo8CcPwhHHqd1g6Qa7uHPh0MkzY8e/6yxVSoFFv2Gtt8MXMyRw+8t3bQULfU8uqLh3mLzhhybMcJwq1cCi/+z3XnGdSG4gYrfZ2LIVLPLW0fri1vojE2eqQJWCYNFJsfeGRFnBok3eUC6nwKLIBKew4AL4QlNL+uqGPFj0sVt9gj4IxI7w2Qks18BiHhavHO/u8zz/7iJ1qFIWLPo7R0Qr4fBsUYhhm0K0CSy2tBP0L1D1MGnxGqWoUhks+ibhcDVDkHVeropgsSRhYSK479myTdtVo0pxsIj5UDQrIlsKfRkIe2r27QBL/ItypLxSQaoUB4v+vdUewTwI3PHi6yelwSJ/AR+diGq/0tP71JS5ebCy6y/uFnINkhtCDNGG3AfLwcJgFLHZIW/q8nVqUuUKsOjbaoSMLYYDt5a7wWIlIvgMhYeKlKXKLWCxf4RgTjOD0t3d7VawdBeDyK5ol7t7HcxcyBmw6E/tEFKIlIsxNG4Fi9gnOziKvHLeuk9UpspFYNH3NwqtEMmosTSD2UKwsK5ECikraxsUp8pdYN1c4A1HhSYtSy0tq8BCA4pMtpFI9LU5y/Ngye1Ly4RchgyQdctDq8Bi3z2RNCAq4tWnynVgfWOpR2RLCP2YDDeBpbvaM/qumK4M7X6WB0u8zzqR2b2OT8u6VC1LwLo00DK+jD1hXEGVG8ESnLRIZLIonUY+WExUZGhk/B6wM4zcnYbyYA3pc4uDIrqFwbIi5UE+WDh2RbwMR1UNC+YMWN9a5hHJBGSwrHCWygeLGKdI8vHkpWvzYFndRSpdSVzmjDvVwcIe5DC+jBU4zR2XVSiRyHmwRm/IXAxNcilDJv1sTslg4cwVWcGu2bbbRVS5Fyz66fbMxHCatfR92ySDxaSaUQ+GwmFl02NyDyzO0XBEG8oEi8WFSKZofxFY3UUmLSs2C82DlejXrfb89lCgtDVz6JAhY+DUBYs6HOKDxjxe7Z2b9xx+dfayPFiy+vUf9PPEbvKGXAiEd+TW8MgEi4qJrL1t9c1tG3cdVDZuqD5Y7F/63vFgRUeWNjieUkN7tNgKFh4R82dSslPtzqMlOCOUWjYqC9bN67xEb0Qs9IzJDpjwKoKF5VRWVpZ5q4+eI1rpD7SOzVooQ4i6o0shwlQDS+eptkuaj4CBw+kg0QUvDSxRA6thirb/D+K96D+05qVaqC39O8gvhTBOD3xownsjGax/XNi/ZQM81XfLPrTui/w5iWaWNLDa2trY9Srz607dOgjWIGH/rrWu1ELt6d/a5/UdLquY9eHHD098b+SA9U8L++7e5FtxKpTFHt2GGsPX3t6uHFj19fWZIzmxoHboL68BVqIf/xetabbmz1D45vH5dcJs2zXEfrASPInvhmWy4c1qaGhQDiyhibT7cDqqkvuRv9MaZ2oBnjOdWL3+QNGZ6iUbt1ldi2EbWJQ1P7LNzybvIkkv7NYnsawZY0ZisrIcsHTLPXO8qWm+KFiJfvgr/WZZ/xyWTtDBULisugbCLNr83Wqw/m1xnKc+IztBisQ5DNnvDKIs+10OWKT1UO4s4JB4yjBYiX7wT7Tzz2h95Vosmj5eBGErP9lh8uRLe8C6/gMv+ekNPbFQJJvhhCq5oZgszgexFixRL0jZD7MHa5CwP9Vqx2meM+kJI5HwXEMTgaNnpy1QDawfrvGurwp1+szODehB0vRU80TKBEvU7jv81xLAGuxf7p/Deo5rsZAIYc9NX+gsWD9a6y2oCHX4ZBrjOJ8kbubOIMrSrXLAIpKTOSAQbJFKVXL/klZ5f7/rNS1hWA91l1oJTb78uyV2gjV6o3f7hXBXwJLFHXWnGN2yrsYgEpdTCCx8DZmLvXpPWAZWUj9zl9a9X4sG5Aa/swDr3s2+XfVhT8haZ4GQl8eI0cYFFQKL5L7MRc+dW+0AK9HLbuoPHEV6RQibuHC1LLAe+NS/tyHsC2fg6UJ33YeVa2cUzzQpeTYek1hmQ/K7oYMdLAcL/0dmJ1bLclvBSvTym/qZjmQo+U0f/M4I1qOF/mOXIsFMkdLG3sa1Veue2PXUTzeNpt+5ZYzJ5T2Zn7JQ0F1Z6FaFwMLXkHmZWj/JGbASvfR6keA352teTdhwYOk8pT/LGXTOd9XA02OfP6nzlNybPc2KoGDAbWQbWOQfZl6b1LzgMFiJfuJbWusHWiiDacIOg5wLTOCI4Hfy3/95Ud+bB/ozM9PvvRGNRSs7qxafWvpA4cNX85ToR5uPmZE8Z8cZTa5M5wUMhWT5L+SAJVKZo537X1XASg5+t2QOfrM9eCox6V4cjoaLW0vnlM6/77Nfp+Ep0bfUbDXpysKrKdH5zlAqBJZQPOfsE8qBlRz8vrRAC2R/UFYwEjrWfPy94lljtv5KhKdE33DuYzOSR1FITFdnEBlKhcA6efJkZiO0+mF1wRoMTf6VSGgyiadgUcuJmcWz7/n0PkM8JXpB1XozkkfsCF8WWBKvZiNYVQ+4AKzk4HfjNM1fd03CPCHvnsZ9k49NvWPLPdnxlOgrK1bnwTKnCqsedBNYid53DdVQ23XBJE+JvqaqIK8KzRnvxPVcR1XdxOGepqB6nRSwdtTvzBvv5twNF95wGVXF3+E7PKgmWi7Rk7XGM3ueNw8WS8i8u8Gcg7RhqqvA+nKKKzUS8T39sO83D2pJ3x9PyDP6k7tMglXfYyobOMcdpEIx9ksL3QTWlRQNFVw633PDf9GDyxYk/724rcQMVXdsuRs/RT6kM7xTXSQI3bHRNVThGUm2PMpLPDd9VwfLc+N3IyXHk/937sn5WYP12sE3TUo+x4PQQmkzfaXuoOro3yfndcW6u7z33BanaqB777oldmXwYSOxSPqgTZpOANG85HM5bUYo0S/c5Q6w+n1Xgya6/80Xk6nSu///ntOS/HbtvvbswDrdYXZBl+OJfsKpyV9RnaqWFSmrpI3rrqZK76GPUyabHfWfG6Xqwe2PRGNma5pzPDVZtJii5HtKU1V+W7KfPVpX47ntB8OB5bnl+uj5lCq88YffMhbMqV5vUuy5X0whukw9+7i6VB38Cy2SpFP8Pt/Ye4alaqD7fn1nLCm9MRAJ3P3pLwWp+vnmOzv9ZucGiSaR3vC1yqqAtbdgtXmxwqGbFEdlYMak9FTpPfDe5JTgQleNIFjTimaYF7sVBauyriatxJ7s5MxWZF+JqqGbCSnLjP27RajSe3j3jtRQz/qMVOFWNZk4qjf0oNwSe4m+VmlgYfdhwmea2ULawT9TjqoT30yufY21t3pH3ygOlvf2Hw0J9Ty95zlLMxriutrvlxgl1JTdFER0G6OT/6MYWH+khTpSQjfPPCJOVdzYempscqinN9h7+yd3DEfV/YUP+sI+8wLHNcCmIBLBUnQbI6p0hKKhjpdUDOmXt6ess5YvNEpVPNSzfFHydU60Fl+Tqp9t/kVpm5yEJ1ZLcrejlbu/re1bRdpTtioauhmbGropHQzdGO2EekqLkq825+S8q8FaV/2RLHtIVrQ42XJXcatIYS9ItH/vK1VCN4OWb6y31/vL27OkSje2xvyE+M/gUEWHhnreOvq2rJFDbREllDh26m5uq4lvx61IKunQ0M04M1TFQz2vpYR62ryDoZ6XDrzqDctRNPoBlhLXg5ri23GL7m/bucV5qpqXpYRuNq03T1U81LMpRdkV1m2HqlcOvCbFYE+skzhiTZPalD5AQPDIk34FdORvHA3d3JIauqlNF7ox2gn11KQc1/hhZQFOeYlCxn0lFwLVjzwx4As+/6wqoZtg0PfofdKo0r0PD43RAn7Nmoa3SfoJ4VxTbmhIc+pYOa23yDGweotTQjczp8ilKh7q+d07VlDFdIVTVGKeTPxrrv6xcvpBmEKJHPi77afqQkrGZvjgHiuoiod69uyUDhbWlfTpyh0HYRrQhpjPDodu2gyFbgx7H35+Q6ylWaJgdUtI1uazyXrQBUf3asKHjWtRv3b0qzaC9Ycpm3/Eov4XnrCOqrj34dlHtUhElmBZCcpK70xuBJ7dcdi4vmwhRJr5pQ2T7QOrc1tK6Gbl+1ZTFQ/1rFosRap4nnG1S1dYDBODJdHhbiFY2kAKPA43gcm9M8MJKBaFbk6dzD50k02o54R5MwibXVZu5xC/qBWzoFVgYQdgDQh9D0iEsjx089WhoZv7breJKt3YuvensR5TuoaSLIk1XkM8+NKNNgvB0gYqDYVMeDaftdrSSj3yyT/+JTup+iLU87yWrbphJYgZZIW2wmyXWOxqE1jM20QJxKbjeRZS1TQnRfd+8nv7qYqHerZsyC5KJjdNdEgYp6+vz2VgaQPngWUuj9YG0kqLvmZN6OZmC0M3RvutQ0M9IhYFVAnJ0HhjJSjxrC9bwcILL5pDTbad/NDNn6dswW1B6MZwqOfhe8VDPXitMNgl1kpc7WWQ7m23CSwDk5ZmwX5/vSlpd4FZU52lKu59mDVNRBik3SE6uRlXQ6YriZsfOQAW3wnR+RbXg8SUh9pXU8bp4F4VqIqHevbuzEgVHma5+exDGoNihVPUPrD0SUt0ym1bKyl0842hoZtf3KQOWP2hntaWNBoQiVlKFSaKpdaVTWCx7sBWEHUZS1CIhG7aUkI3Lz6pDlVx78Nzj2nXEoheUG6RxzLhu2I4rFsM2geWHuQSNRcwt5lvTIVuUvbjD61aohpVce/DqiXX9Cxk3g3KXINaK0LOzoCFG0YoszQu4NP9uXjZUVX5qxRKT5fZF7rJItRzcjAzDP+n0OZ1MgbCIq+YA2BpA9WVBhKJsjsnbEjops/u0I3JUI8VvvWrQ0PWrTSdAQupYZNiNgqHhMYZBsuXAm7g7ddVpiqeaDrhFc2uhvAZAhvwtRWshA0hvEtYzJghnxq6CW/ZqD5Vce/Dpx/bIHzELr0EQxWwtIF0GgNRTzIBBU+95zDV5INJKBKc8LJbwOJWNVuUoKWLTYfBYh7Gg5J5U5rBRd1lrfjbGag68Mea7xolvOF9u4ZsSqucjTXmJ9ykDWJnqw/EbpsSdAAs7YtULaH80jggV7TS69KB1fjusBx7PcH35zgZeB6u3/bfwcVzY7YoJo6usC7pSiGwaLhqcANGxJPB2W6ZU3evrQRv1GIZrhPrvhJcMo/kAiWQ+vF1FJzFOjvsETVCRtQG1kyuBotGyMJYitk12Tryt1pQdLsBoijBRbO9o29wTPGNviH4/uxYW4udcsY1LZQjnjNgoe9J2zC2CwXlyxX3JIH1Je3yNsMf7PeFt27yPX6/rdkyT9wf/myTdbXRaZzsBLNtNq0cBktfADNLG4xgxLT6t/uRAiwcXWbIbmlmo3aG3EKeHhpD3CbaWOeIeFF/iFfiFvCuAUsbqD3CrjRcfNK+Xiv9fr8zQsrc2dwU/mxzYNLrHGQiQd/dfWtg8htckMs6KFhizHit7DfYVQFLFwFssXIxiEPEipshwSZy7FBozQrg8D/zaP8+bD++Lo0ZTsiIklReHFq7MnLsMG/XFGh4QRGpDfkLSoOlDWQziha4OmAMRikXQ29Gmy5Gz1b196aL/MoftVhUwfvV0+QtzTl2DVi6A4IkIWen7hxoOlWOOBcUBUsbOL0DthSdt9zQ9GJ564ov3AqWvpDBOLAzUJozDaHZkCToVrC0geILFQxPdzWW1QjN6uIId4Ol2ZWhmzMNQan5VRyloLD0U2Lk7g6dkw3fOl5QNRc9o9QUGS5jYj7EEyORSB6gqxtiQThEbBz0rbsSLG0gnkismm9kfqk4pOFPRiwIx6k4oLvBSrYhDOQG5npDFAhEEWeVi8HSBoqWUIsU+YgWkOVoQ+uh/sgFdYUneZQrZMqcT768Om5l+xsPzuNjraus/twHVsK24PvK1GVPyaUijXmaiYrKLXe5jke5S8p8Xym5xMjAGSF9C2HVGg/IY/KwPLJbJiq3gpWwuki6xdelQhjfOt3HA7LPgkun51HuFT3u5uqBZvWWBzY3fVc0nsvVoa1Rbh8GJi2GgWVjDuAFUjwIj6Na4G8kgpVQHHzF2QaYvBHXmSPYUtw2N88j5IxyzxGwEsqRNSPWLjVPrvDXc5PcKjfMbedYTkdOgaU3/Ie604tQGn5qBaNp3BI3htbjJrnVnEydzUGwEo4JLBVWVRzGx0GPDKTjyytugNvgZrglbozbc53WzoOVYsFguDCQaBzsGFQPZr5tSRN8EB/Hh/LR3AC3wc3kvAduRICV3HBe43JkzigrK2Px1djYyBRCaqFEzrgUF+SyXByY+CA+jg8daSnXIwusZEXJSLO/T0NDA2sxhh9vJDYZvwIBKRXoKfjA+sEeigy0BDc0/sh/8QJexot5C2/k7VyES3FBfuXifEQOK7s8WKImP8njLPsJ9HKkO+E5jGvSntBfmETgUjrQ+IFf+SP/paci8mLewht5e76CLdH+H5nMNBUt+3p0AAAAAElFTkSuQmCC", + "description": "Displays the latest values of the attributes or time series data for multiple entities in a polar area chart. Supports numeric values only.", + "descriptor": { + "type": "latest", + "sizeX": 7, + "sizeY": 5, + "resources": [ + { + "url": "https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.3.0/Chart.min.js" + } + ], + "templateHtml": "\n", + "templateCss": "", + "controllerScript": "self.onInit = function() {\n $scope = self.ctx.$scope;\n utils = $scope.$injector.get(self.ctx.servicesMap.get('utils'));\n settings = utils.deepClone(self.ctx.settings) || {};\n settings.showTooltip = utils.defaultValue(settings.showTooltip, true);\n \n Chart.defaults.global.tooltips.enabled = settings.showTooltip;\n \n var pieData = {\n labels: [],\n datasets: []\n };\n\n var dataset = {\n data: [],\n backgroundColor: [],\n borderColor: [],\n borderWidth: [],\n hoverBackgroundColor: []\n }\n\n pieData.datasets.push(dataset);\n \n for (var i = 0; i < self.ctx.data.length; i++) {\n var dataKey = self.ctx.data[i].dataKey;\n var units = dataKey.units && dataKey.units.length ? dataKey.units : self.ctx.units;\n units = units ? (' (' + units + ')') : '';\n pieData.labels.push(dataKey.label + units);\n dataset.data.push(0);\n var hoverBackgroundColor = tinycolor(dataKey.color).lighten(15);\n var borderColor = tinycolor(dataKey.color).darken();\n dataset.backgroundColor.push(dataKey.color);\n dataset.borderColor.push('#fff');\n dataset.borderWidth.push(5);\n dataset.hoverBackgroundColor.push(hoverBackgroundColor.toRgbString());\n }\n \n var floatingPoint;\n if (typeof self.ctx.decimals !== 'undefined' && self.ctx.decimals !== null) {\n floatingPoint = self.ctx.widget.config.decimals;\n } else {\n floatingPoint = 2;\n }\n\n\n var ctx = $('#pieChart', self.ctx.$container);\n self.ctx.chart = new Chart(ctx, {\n type: 'polarArea',\n data: pieData,\n options: {\n responsive: false,\n maintainAspectRatio: false,\n scale: {\n ticks: {\n callback: function(tick) {\n \treturn tick.toFixed(floatingPoint);\n }\n }\n }\n }\n });\n \n self.onResize();\n}\n\nself.onDataUpdated = function() {\n for (var i = 0; i < self.ctx.data.length; i++) {\n var cellData = self.ctx.data[i];\n if (cellData.data.length > 0) {\n var decimals;\n if (typeof cellData.dataKey.decimals !== 'undefined' \n && cellData.dataKey.decimals !== null ) {\n decimals = cellData.dataKey.decimals; \n } else {\n decimals = self.ctx.decimals;\n }\n var tvPair = cellData.data[cellData.data.length - 1];\n var value = self.ctx.utils.formatValue(tvPair[1], decimals);\n self.ctx.chart.data.datasets[0].data[i] = parseFloat(value);\n }\n }\n self.ctx.chart.update();\n}\n\nself.onResize = function() {\n if (self.ctx.height >= 70) {\n try {\n self.ctx.chart.resize();\n } catch (e) {}\n }\n}\n\nself.onDestroy = function() {\n self.ctx.chart.destroy();\n self.ctx.chart = null;\n}\n", + "settingsSchema": "", + "dataKeySettingsSchema": "{}\n", + "settingsDirective": "tb-chart-widget-settings", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"First\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.15479322438769105,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Second\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.545701115289893,\"funcBody\":\"var value = (prevValue-20) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+20;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Third\",\"color\":\"#f44336\",\"settings\":{},\"_hash\":0.2592906835158064,\"funcBody\":\"var value = (prevValue-40) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+40;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fourth\",\"color\":\"#ffc107\",\"settings\":{},\"_hash\":0.12880275585455747,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"Fifth\",\"color\":\"#607d8b\",\"settings\":{},\"_hash\":0.2074391823443591,\"funcBody\":\"var value = (prevValue-50) + Math.random() * 2 - 1;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 100) {\\n\\tvalue = 100;\\n}\\nreturn value+50;\"}]}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":true,\"backgroundColor\":\"#fff\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"8px\",\"settings\":{},\"title\":\"Polar Area\"}" + }, + "externalId": null +} \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts index 9e7c91c114..0ce07406c9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts @@ -120,6 +120,12 @@ import { import { PieChartBasicConfigComponent } from '@home/components/widget/config/basic/chart/pie-chart-basic-config.component'; +import { + BarChartBasicConfigComponent +} from '@home/components/widget/config/basic/chart/bar-chart-basic-config.component'; +import { + PolarAreaChartBasicConfigComponent +} from '@home/components/widget/config/basic/chart/polar-area-chart-basic-config.component'; @NgModule({ declarations: [ @@ -159,7 +165,9 @@ import { ComparisonKeyRowComponent, ComparisonKeysTableComponent, StatusWidgetBasicConfigComponent, - PieChartBasicConfigComponent + PieChartBasicConfigComponent, + BarChartBasicConfigComponent, + PolarAreaChartBasicConfigComponent ], imports: [ CommonModule, @@ -201,7 +209,9 @@ import { ToggleButtonBasicConfigComponent, TimeSeriesChartBasicConfigComponent, StatusWidgetBasicConfigComponent, - PieChartBasicConfigComponent + PieChartBasicConfigComponent, + BarChartBasicConfigComponent, + PolarAreaChartBasicConfigComponent ] }) export class BasicWidgetConfigModule { @@ -237,5 +247,7 @@ export const basicWidgetConfigComponentsMap: {[key: string]: Type + + + + + + + +
+
widget-config.card-appearance
+
+ + {{ 'widget-config.card-title' | translate }} + +
+ + + + + + + +
+
+
+ + {{ 'widget-config.card-icon' | translate }} + +
+ + + + +
+
+
+ + {{ 'widgets.latest-chart.sort-series' | translate }} + +
+
+
widget-config.units-short
+ + +
+
+
widget-config.decimals-short
+ + + +
+
+
+
widgets.bar-chart.bar-appearance
+ + +
+
+
widgets.bar-chart.bar-axis
+
+
widgets.chart.chart-axis.scale
+
+
widgets.chart.chart-axis.scale-min
+ + + +
widgets.chart.chart-axis.scale-max
+ + + + + + + +
+
+
+
+ + + + + {{ 'widget-config.legend' | translate }} + + + + +
+
{{ 'legend.position' | translate }}
+ + + + {{ legendPositionTranslationMap.get(pos) | translate }} + + + +
+
+
{{ 'legend.label' | translate }}
+
+ + + + +
+
+
+
{{ 'legend.value' | translate }}
+
+ + + + +
+
+ + +
+
+ + + + + {{ 'widget-config.tooltip' | translate }} + + + + +
+
{{ 'tooltip.value' | translate }}
+
+ + + + {{ latestChartTooltipValueTypeTranslationMap.get(type) | translate }} + + + + + +
widget-config.decimals-suffix
+ + + + + +
+
+
+
{{ 'tooltip.background-color' | translate }}
+ + +
+
+
{{ 'tooltip.background-blur' | translate }}
+ + +
px
+ +
+ + +
+ + +
+
widget-config.card-appearance
+
+
{{ 'widgets.background.background' | translate }}
+ + +
+
+
widget-config.show-card-buttons
+ + {{ 'fullscreen.fullscreen' | translate }} + +
+
+
{{ 'widget-config.card-border-radius' | translate }}
+ + + +
+
+ + + diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-basic-config.component.ts new file mode 100644 index 0000000000..da8b94410d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-basic-config.component.ts @@ -0,0 +1,316 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { BasicWidgetConfigComponent } from '@home/components/widget/config/widget-config.component.models'; +import { WidgetConfigComponentData } from '@home/models/widget-component.models'; +import { + DataKey, + Datasource, + datasourcesHasAggregation, + datasourcesHasOnlyComparisonAggregation, + legendPositions, + legendPositionTranslationMap, + WidgetConfig +} from '@shared/models/widget.models'; +import { WidgetConfigComponent } from '@home/components/widget/widget-config.component'; +import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { formatValue, isUndefined, mergeDeep } from '@core/utils'; +import { + getTimewindowConfig, + setTimewindowConfig +} from '@home/components/widget/config/timewindow-config-panel.component'; +import { + LatestChartTooltipValueType, + latestChartTooltipValueTypes, + latestChartTooltipValueTypeTranslations +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { + barChartWidgetDefaultSettings, + BarChartWidgetSettings +} from '@home/components/widget/lib/chart/bar-chart-widget.models'; + +@Component({ + selector: 'tb-bar-chart-basic-config', + templateUrl: './bar-chart-basic-config.component.html', + styleUrls: ['../basic-config.scss'] +}) +export class BarChartBasicConfigComponent extends BasicWidgetConfigComponent { + + public get datasource(): Datasource { + const datasources: Datasource[] = this.barChartWidgetConfigForm.get('datasources').value; + if (datasources && datasources.length) { + return datasources[0]; + } else { + return null; + } + } + + public get displayTimewindowConfig(): boolean { + const datasources = this.barChartWidgetConfigForm.get('datasources').value; + return datasourcesHasAggregation(datasources); + } + + public onlyHistoryTimewindow(): boolean { + const datasources = this.barChartWidgetConfigForm.get('datasources').value; + return datasourcesHasOnlyComparisonAggregation(datasources); + } + + legendPositions = legendPositions; + + legendPositionTranslationMap = legendPositionTranslationMap; + + latestChartTooltipValueTypes = latestChartTooltipValueTypes; + + latestChartTooltipValueTypeTranslationMap = latestChartTooltipValueTypeTranslations; + + barChartWidgetConfigForm: UntypedFormGroup; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + tooltipValuePreviewFn = this._tooltipValuePreviewFn.bind(this); + + constructor(protected store: Store, + protected widgetConfigComponent: WidgetConfigComponent, + private fb: UntypedFormBuilder) { + super(store, widgetConfigComponent); + } + + protected configForm(): UntypedFormGroup { + return this.barChartWidgetConfigForm; + } + + protected defaultDataKeys(configData: WidgetConfigComponentData): DataKey[] { + return [{ name: 'windPower', label: 'Wind', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#08872B' }, + { name: 'solarPower', label: 'Solar', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#FF4D5A' }, + { name: 'hydroelectricPower', label: 'Hydroelectric', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#FFDE30' }]; + } + + protected onConfigSet(configData: WidgetConfigComponentData) { + const settings: BarChartWidgetSettings = mergeDeep({} as BarChartWidgetSettings, + barChartWidgetDefaultSettings, configData.config.settings as BarChartWidgetSettings); + this.barChartWidgetConfigForm = this.fb.group({ + timewindowConfig: [getTimewindowConfig(configData.config), []], + datasources: [configData.config.datasources, []], + + series: [this.getSeries(configData.config.datasources), []], + + showTitle: [configData.config.showTitle, []], + title: [configData.config.title, []], + titleFont: [configData.config.titleFont, []], + titleColor: [configData.config.titleColor, []], + showTitleIcon: [configData.config.showTitleIcon, []], + titleIcon: [configData.config.titleIcon, []], + iconColor: [configData.config.iconColor, []], + + sortSeries: [settings.sortSeries, []], + + units: [configData.config.units, []], + decimals: [configData.config.decimals, []], + + barSettings: [settings.barSettings, []], + + axisMin: [settings.axisMin, []], + axisMax: [settings.axisMax, []], + axisTickLabelFont: [settings.axisTickLabelFont, []], + axisTickLabelColor: [settings.axisTickLabelColor, []], + + animation: [settings.animation, []], + + showLegend: [settings.showLegend, []], + legendPosition: [settings.legendPosition, []], + legendLabelFont: [settings.legendLabelFont, []], + legendLabelColor: [settings.legendLabelColor, []], + legendValueFont: [settings.legendValueFont, []], + legendValueColor: [settings.legendValueColor, []], + + showTooltip: [settings.showTooltip, []], + tooltipValueType: [settings.tooltipValueType, []], + tooltipValueDecimals: [settings.tooltipValueDecimals, []], + tooltipValueFont: [settings.tooltipValueFont, []], + tooltipValueColor: [settings.tooltipValueColor, []], + tooltipBackgroundColor: [settings.tooltipBackgroundColor, []], + tooltipBackgroundBlur: [settings.tooltipBackgroundBlur, []], + + background: [settings.background, []], + + cardButtons: [this.getCardButtons(configData.config), []], + borderRadius: [configData.config.borderRadius, []], + + actions: [configData.config.actions || {}, []] + }); + } + + protected prepareOutputConfig(config: any): WidgetConfigComponentData { + setTimewindowConfig(this.widgetConfig.config, config.timewindowConfig); + this.widgetConfig.config.datasources = config.datasources; + this.setSeries(config.series, this.widgetConfig.config.datasources); + + this.widgetConfig.config.showTitle = config.showTitle; + this.widgetConfig.config.title = config.title; + this.widgetConfig.config.titleFont = config.titleFont; + this.widgetConfig.config.titleColor = config.titleColor; + + this.widgetConfig.config.showTitleIcon = config.showTitleIcon; + this.widgetConfig.config.titleIcon = config.titleIcon; + this.widgetConfig.config.iconColor = config.iconColor; + + this.widgetConfig.config.settings = this.widgetConfig.config.settings || {}; + + this.widgetConfig.config.settings.sortSeries = config.sortSeries; + + this.widgetConfig.config.units = config.units; + this.widgetConfig.config.decimals = config.decimals; + + this.widgetConfig.config.settings.barSettings = config.barSettings; + + this.widgetConfig.config.settings.axisMin = config.axisMin; + this.widgetConfig.config.settings.axisMax = config.axisMax; + this.widgetConfig.config.settings.axisTickLabelFont = config.axisTickLabelFont; + this.widgetConfig.config.settings.axisTickLabelColor = config.axisTickLabelColor; + + this.widgetConfig.config.settings.animation = config.animation; + + this.widgetConfig.config.settings.showLegend = config.showLegend; + this.widgetConfig.config.settings.legendPosition = config.legendPosition; + this.widgetConfig.config.settings.legendLabelFont = config.legendLabelFont; + this.widgetConfig.config.settings.legendLabelColor = config.legendLabelColor; + this.widgetConfig.config.settings.legendValueFont = config.legendValueFont; + this.widgetConfig.config.settings.legendValueColor = config.legendValueColor; + + this.widgetConfig.config.settings.showTooltip = config.showTooltip; + this.widgetConfig.config.settings.tooltipValueType = config.tooltipValueType; + this.widgetConfig.config.settings.tooltipValueDecimals = config.tooltipValueDecimals; + this.widgetConfig.config.settings.tooltipValueFont = config.tooltipValueFont; + this.widgetConfig.config.settings.tooltipValueColor = config.tooltipValueColor; + this.widgetConfig.config.settings.tooltipBackgroundColor = config.tooltipBackgroundColor; + this.widgetConfig.config.settings.tooltipBackgroundBlur = config.tooltipBackgroundBlur; + + this.widgetConfig.config.settings.background = config.background; + + this.setCardButtons(config.cardButtons, this.widgetConfig.config); + this.widgetConfig.config.borderRadius = config.borderRadius; + + this.widgetConfig.config.actions = config.actions; + + return this.widgetConfig; + } + + protected validatorTriggers(): string[] { + return ['showTitle', 'showTitleIcon', 'showLegend', 'showTooltip']; + } + + protected updateValidators(emitEvent: boolean, trigger?: string) { + const showTitle: boolean = this.barChartWidgetConfigForm.get('showTitle').value; + const showTitleIcon: boolean = this.barChartWidgetConfigForm.get('showTitleIcon').value; + const showLegend: boolean = this.barChartWidgetConfigForm.get('showLegend').value; + const showTooltip: boolean = this.barChartWidgetConfigForm.get('showTooltip').value; + + if (showTitle) { + this.barChartWidgetConfigForm.get('title').enable(); + this.barChartWidgetConfigForm.get('titleFont').enable(); + this.barChartWidgetConfigForm.get('titleColor').enable(); + this.barChartWidgetConfigForm.get('showTitleIcon').enable({emitEvent: false}); + if (showTitleIcon) { + this.barChartWidgetConfigForm.get('titleIcon').enable(); + this.barChartWidgetConfigForm.get('iconColor').enable(); + } else { + this.barChartWidgetConfigForm.get('titleIcon').disable(); + this.barChartWidgetConfigForm.get('iconColor').disable(); + } + } else { + this.barChartWidgetConfigForm.get('title').disable(); + this.barChartWidgetConfigForm.get('titleFont').disable(); + this.barChartWidgetConfigForm.get('titleColor').disable(); + this.barChartWidgetConfigForm.get('showTitleIcon').disable({emitEvent: false}); + this.barChartWidgetConfigForm.get('titleIcon').disable(); + this.barChartWidgetConfigForm.get('iconColor').disable(); + } + if (showLegend) { + this.barChartWidgetConfigForm.get('legendPosition').enable(); + this.barChartWidgetConfigForm.get('legendLabelFont').enable(); + this.barChartWidgetConfigForm.get('legendLabelColor').enable(); + this.barChartWidgetConfigForm.get('legendValueFont').enable(); + this.barChartWidgetConfigForm.get('legendValueColor').enable(); + } else { + this.barChartWidgetConfigForm.get('legendPosition').disable(); + this.barChartWidgetConfigForm.get('legendLabelFont').disable(); + this.barChartWidgetConfigForm.get('legendLabelColor').disable(); + this.barChartWidgetConfigForm.get('legendValueFont').disable(); + this.barChartWidgetConfigForm.get('legendValueColor').disable(); + } + if (showTooltip) { + this.barChartWidgetConfigForm.get('tooltipValueType').enable(); + this.barChartWidgetConfigForm.get('tooltipValueDecimals').enable(); + this.barChartWidgetConfigForm.get('tooltipValueFont').enable(); + this.barChartWidgetConfigForm.get('tooltipValueColor').enable(); + this.barChartWidgetConfigForm.get('tooltipBackgroundColor').enable(); + this.barChartWidgetConfigForm.get('tooltipBackgroundBlur').enable(); + } else { + this.barChartWidgetConfigForm.get('tooltipValueType').disable(); + this.barChartWidgetConfigForm.get('tooltipValueDecimals').disable(); + this.barChartWidgetConfigForm.get('tooltipValueFont').disable(); + this.barChartWidgetConfigForm.get('tooltipValueColor').disable(); + this.barChartWidgetConfigForm.get('tooltipBackgroundColor').disable(); + this.barChartWidgetConfigForm.get('tooltipBackgroundBlur').disable(); + } + } + + private getSeries(datasources?: Datasource[]): DataKey[] { + if (datasources && datasources.length) { + return datasources[0].dataKeys || []; + } + return []; + } + + private setSeries(series: DataKey[], datasources?: Datasource[]) { + if (datasources && datasources.length) { + datasources[0].dataKeys = series; + } + } + + private getCardButtons(config: WidgetConfig): string[] { + const buttons: string[] = []; + if (isUndefined(config.enableFullscreen) || config.enableFullscreen) { + buttons.push('fullscreen'); + } + return buttons; + } + + private setCardButtons(buttons: string[], config: WidgetConfig) { + config.enableFullscreen = buttons.includes('fullscreen'); + } + + private _valuePreviewFn(): string { + const units: string = this.barChartWidgetConfigForm.get('units').value; + const decimals: number = this.barChartWidgetConfigForm.get('decimals').value; + return formatValue(110, decimals, units, false); + } + + private _tooltipValuePreviewFn(): string { + const tooltipValueType: LatestChartTooltipValueType = this.barChartWidgetConfigForm.get('tooltipValueType').value; + const decimals: number = this.barChartWidgetConfigForm.get('tooltipValueDecimals').value; + if (tooltipValueType === LatestChartTooltipValueType.percentage) { + return formatValue(35, decimals, '%', false); + } else { + const units: string = this.barChartWidgetConfigForm.get('units').value; + return formatValue(110, decimals, units, false); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.html index 458dfffa14..0d8537a3a6 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/bar-chart-with-labels-basic-config.component.html @@ -143,26 +143,26 @@
- {{ 'widgets.time-series-chart.series.bar.show-border' | translate }} + {{ 'widgets.chart.bar.show-border' | translate }}
-
widgets.time-series-chart.series.bar.border-width
+
widgets.chart.bar.border-width
-
widgets.time-series-chart.series.bar.border-radius
+
widgets.chart.bar.border-radius
- - + title="widgets.chart.background" + fillNoneTitle="widgets.chart.fill-type-solid"> + @@ -306,9 +306,9 @@ - - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/doughnut-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/doughnut-basic-config.component.html index 8ba255052d..13a4985647 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/doughnut-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/doughnut-basic-config.component.html @@ -229,9 +229,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.html index e0b6352a8a..bdc94b1479 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.html @@ -247,9 +247,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.ts index 7e0dc8d993..e8d34933c3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/pie-chart-basic-config.component.ts @@ -48,7 +48,7 @@ import { import { pieChartLabelPositions, pieChartLabelPositionTranslations -} from '@home/components/widget/lib/chart/pie-chart.models'; +} from '@home/components/widget/lib/chart/chart.models'; @Component({ selector: 'tb-pie-chart-basic-config', diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.html new file mode 100644 index 0000000000..92b94b5912 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.html @@ -0,0 +1,268 @@ + + + + + + + + +
+
widget-config.card-appearance
+
+ + {{ 'widget-config.card-title' | translate }} + +
+ + + + + + + +
+
+
+ + {{ 'widget-config.card-icon' | translate }} + +
+ + + + +
+
+
+ + {{ 'widgets.latest-chart.sort-series' | translate }} + +
+
+
widget-config.units-short
+ + +
+
+
widget-config.decimals-short
+ + + +
+
+
+
widgets.bar-chart.bar-appearance
+ + +
+
+
widgets.polar-area-chart.polar-axis
+
+
widgets.chart.chart-axis.scale
+
+
widgets.chart.chart-axis.scale-min
+ + + +
widgets.chart.chart-axis.scale-max
+ + + + + + + +
+
+
+
widgets.polar-area-chart.start-angle
+ + + +
+
+
+ + + + + {{ 'widget-config.legend' | translate }} + + + + +
+
{{ 'legend.position' | translate }}
+ + + + {{ legendPositionTranslationMap.get(pos) | translate }} + + + +
+
+
{{ 'legend.label' | translate }}
+
+ + + + +
+
+
+
{{ 'legend.value' | translate }}
+
+ + + + +
+
+ + +
+
+ + + + + {{ 'widget-config.tooltip' | translate }} + + + + +
+
{{ 'tooltip.value' | translate }}
+
+ + + + {{ latestChartTooltipValueTypeTranslationMap.get(type) | translate }} + + + + + +
widget-config.decimals-suffix
+ + + + + +
+
+
+
{{ 'tooltip.background-color' | translate }}
+ + +
+
+
{{ 'tooltip.background-blur' | translate }}
+ + +
px
+ +
+ + +
+ + +
+
widget-config.card-appearance
+
+
{{ 'widgets.background.background' | translate }}
+ + +
+
+
widget-config.show-card-buttons
+ + {{ 'fullscreen.fullscreen' | translate }} + +
+
+
{{ 'widget-config.card-border-radius' | translate }}
+ + + +
+
+ + + diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.ts new file mode 100644 index 0000000000..0d195dde38 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/polar-area-chart-basic-config.component.ts @@ -0,0 +1,318 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { BasicWidgetConfigComponent } from '@home/components/widget/config/widget-config.component.models'; +import { WidgetConfigComponentData } from '@home/models/widget-component.models'; +import { + DataKey, + Datasource, + datasourcesHasAggregation, + datasourcesHasOnlyComparisonAggregation, + legendPositions, + legendPositionTranslationMap, + WidgetConfig +} from '@shared/models/widget.models'; +import { WidgetConfigComponent } from '@home/components/widget/widget-config.component'; +import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; +import { formatValue, isUndefined, mergeDeep } from '@core/utils'; +import { + getTimewindowConfig, + setTimewindowConfig +} from '@home/components/widget/config/timewindow-config-panel.component'; +import { + LatestChartTooltipValueType, + latestChartTooltipValueTypes, + latestChartTooltipValueTypeTranslations +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { + polarAreaChartWidgetDefaultSettings, + PolarAreaChartWidgetSettings +} from '@home/components/widget/lib/chart/polar-area-widget.models'; + +@Component({ + selector: 'tb-polar-area-chart-basic-config', + templateUrl: './polar-area-chart-basic-config.component.html', + styleUrls: ['../basic-config.scss'] +}) +export class PolarAreaChartBasicConfigComponent extends BasicWidgetConfigComponent { + + public get datasource(): Datasource { + const datasources: Datasource[] = this.polarAreaChartWidgetConfigForm.get('datasources').value; + if (datasources && datasources.length) { + return datasources[0]; + } else { + return null; + } + } + + public get displayTimewindowConfig(): boolean { + const datasources = this.polarAreaChartWidgetConfigForm.get('datasources').value; + return datasourcesHasAggregation(datasources); + } + + public onlyHistoryTimewindow(): boolean { + const datasources = this.polarAreaChartWidgetConfigForm.get('datasources').value; + return datasourcesHasOnlyComparisonAggregation(datasources); + } + + legendPositions = legendPositions; + + legendPositionTranslationMap = legendPositionTranslationMap; + + latestChartTooltipValueTypes = latestChartTooltipValueTypes; + + latestChartTooltipValueTypeTranslationMap = latestChartTooltipValueTypeTranslations; + + polarAreaChartWidgetConfigForm: UntypedFormGroup; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + tooltipValuePreviewFn = this._tooltipValuePreviewFn.bind(this); + + constructor(protected store: Store, + protected widgetConfigComponent: WidgetConfigComponent, + private fb: UntypedFormBuilder) { + super(store, widgetConfigComponent); + } + + protected configForm(): UntypedFormGroup { + return this.polarAreaChartWidgetConfigForm; + } + + protected defaultDataKeys(configData: WidgetConfigComponentData): DataKey[] { + return [{ name: 'windPower', label: 'Wind', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#08872B' }, + { name: 'solarPower', label: 'Solar', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#FF4D5A' }, + { name: 'hydroelectricPower', label: 'Hydroelectric', type: DataKeyType.timeseries, units: '', decimals: 0, color: '#FFDE30' }]; + } + + protected onConfigSet(configData: WidgetConfigComponentData) { + const settings: PolarAreaChartWidgetSettings = mergeDeep({} as PolarAreaChartWidgetSettings, + polarAreaChartWidgetDefaultSettings, configData.config.settings as PolarAreaChartWidgetSettings); + this.polarAreaChartWidgetConfigForm = this.fb.group({ + timewindowConfig: [getTimewindowConfig(configData.config), []], + datasources: [configData.config.datasources, []], + + series: [this.getSeries(configData.config.datasources), []], + + showTitle: [configData.config.showTitle, []], + title: [configData.config.title, []], + titleFont: [configData.config.titleFont, []], + titleColor: [configData.config.titleColor, []], + showTitleIcon: [configData.config.showTitleIcon, []], + titleIcon: [configData.config.titleIcon, []], + iconColor: [configData.config.iconColor, []], + + sortSeries: [settings.sortSeries, []], + + units: [configData.config.units, []], + decimals: [configData.config.decimals, []], + + barSettings: [settings.barSettings, []], + + axisMin: [settings.axisMin, []], + axisMax: [settings.axisMax, []], + axisTickLabelFont: [settings.axisTickLabelFont, []], + axisTickLabelColor: [settings.axisTickLabelColor, []], + angleAxisStartAngle: [settings.angleAxisStartAngle, [Validators.min(0), Validators.max(360)]], + + animation: [settings.animation, []], + + showLegend: [settings.showLegend, []], + legendPosition: [settings.legendPosition, []], + legendLabelFont: [settings.legendLabelFont, []], + legendLabelColor: [settings.legendLabelColor, []], + legendValueFont: [settings.legendValueFont, []], + legendValueColor: [settings.legendValueColor, []], + + showTooltip: [settings.showTooltip, []], + tooltipValueType: [settings.tooltipValueType, []], + tooltipValueDecimals: [settings.tooltipValueDecimals, []], + tooltipValueFont: [settings.tooltipValueFont, []], + tooltipValueColor: [settings.tooltipValueColor, []], + tooltipBackgroundColor: [settings.tooltipBackgroundColor, []], + tooltipBackgroundBlur: [settings.tooltipBackgroundBlur, []], + + background: [settings.background, []], + + cardButtons: [this.getCardButtons(configData.config), []], + borderRadius: [configData.config.borderRadius, []], + + actions: [configData.config.actions || {}, []] + }); + } + + protected prepareOutputConfig(config: any): WidgetConfigComponentData { + setTimewindowConfig(this.widgetConfig.config, config.timewindowConfig); + this.widgetConfig.config.datasources = config.datasources; + this.setSeries(config.series, this.widgetConfig.config.datasources); + + this.widgetConfig.config.showTitle = config.showTitle; + this.widgetConfig.config.title = config.title; + this.widgetConfig.config.titleFont = config.titleFont; + this.widgetConfig.config.titleColor = config.titleColor; + + this.widgetConfig.config.showTitleIcon = config.showTitleIcon; + this.widgetConfig.config.titleIcon = config.titleIcon; + this.widgetConfig.config.iconColor = config.iconColor; + + this.widgetConfig.config.settings = this.widgetConfig.config.settings || {}; + + this.widgetConfig.config.settings.sortSeries = config.sortSeries; + + this.widgetConfig.config.units = config.units; + this.widgetConfig.config.decimals = config.decimals; + + this.widgetConfig.config.settings.barSettings = config.barSettings; + + this.widgetConfig.config.settings.axisMin = config.axisMin; + this.widgetConfig.config.settings.axisMax = config.axisMax; + this.widgetConfig.config.settings.axisTickLabelFont = config.axisTickLabelFont; + this.widgetConfig.config.settings.axisTickLabelColor = config.axisTickLabelColor; + this.widgetConfig.config.settings.angleAxisStartAngle = config.angleAxisStartAngle; + + this.widgetConfig.config.settings.animation = config.animation; + + this.widgetConfig.config.settings.showLegend = config.showLegend; + this.widgetConfig.config.settings.legendPosition = config.legendPosition; + this.widgetConfig.config.settings.legendLabelFont = config.legendLabelFont; + this.widgetConfig.config.settings.legendLabelColor = config.legendLabelColor; + this.widgetConfig.config.settings.legendValueFont = config.legendValueFont; + this.widgetConfig.config.settings.legendValueColor = config.legendValueColor; + + this.widgetConfig.config.settings.showTooltip = config.showTooltip; + this.widgetConfig.config.settings.tooltipValueType = config.tooltipValueType; + this.widgetConfig.config.settings.tooltipValueDecimals = config.tooltipValueDecimals; + this.widgetConfig.config.settings.tooltipValueFont = config.tooltipValueFont; + this.widgetConfig.config.settings.tooltipValueColor = config.tooltipValueColor; + this.widgetConfig.config.settings.tooltipBackgroundColor = config.tooltipBackgroundColor; + this.widgetConfig.config.settings.tooltipBackgroundBlur = config.tooltipBackgroundBlur; + + this.widgetConfig.config.settings.background = config.background; + + this.setCardButtons(config.cardButtons, this.widgetConfig.config); + this.widgetConfig.config.borderRadius = config.borderRadius; + + this.widgetConfig.config.actions = config.actions; + + return this.widgetConfig; + } + + protected validatorTriggers(): string[] { + return ['showTitle', 'showTitleIcon', 'showLegend', 'showTooltip']; + } + + protected updateValidators(emitEvent: boolean, trigger?: string) { + const showTitle: boolean = this.polarAreaChartWidgetConfigForm.get('showTitle').value; + const showTitleIcon: boolean = this.polarAreaChartWidgetConfigForm.get('showTitleIcon').value; + const showLegend: boolean = this.polarAreaChartWidgetConfigForm.get('showLegend').value; + const showTooltip: boolean = this.polarAreaChartWidgetConfigForm.get('showTooltip').value; + + if (showTitle) { + this.polarAreaChartWidgetConfigForm.get('title').enable(); + this.polarAreaChartWidgetConfigForm.get('titleFont').enable(); + this.polarAreaChartWidgetConfigForm.get('titleColor').enable(); + this.polarAreaChartWidgetConfigForm.get('showTitleIcon').enable({emitEvent: false}); + if (showTitleIcon) { + this.polarAreaChartWidgetConfigForm.get('titleIcon').enable(); + this.polarAreaChartWidgetConfigForm.get('iconColor').enable(); + } else { + this.polarAreaChartWidgetConfigForm.get('titleIcon').disable(); + this.polarAreaChartWidgetConfigForm.get('iconColor').disable(); + } + } else { + this.polarAreaChartWidgetConfigForm.get('title').disable(); + this.polarAreaChartWidgetConfigForm.get('titleFont').disable(); + this.polarAreaChartWidgetConfigForm.get('titleColor').disable(); + this.polarAreaChartWidgetConfigForm.get('showTitleIcon').disable({emitEvent: false}); + this.polarAreaChartWidgetConfigForm.get('titleIcon').disable(); + this.polarAreaChartWidgetConfigForm.get('iconColor').disable(); + } + if (showLegend) { + this.polarAreaChartWidgetConfigForm.get('legendPosition').enable(); + this.polarAreaChartWidgetConfigForm.get('legendLabelFont').enable(); + this.polarAreaChartWidgetConfigForm.get('legendLabelColor').enable(); + this.polarAreaChartWidgetConfigForm.get('legendValueFont').enable(); + this.polarAreaChartWidgetConfigForm.get('legendValueColor').enable(); + } else { + this.polarAreaChartWidgetConfigForm.get('legendPosition').disable(); + this.polarAreaChartWidgetConfigForm.get('legendLabelFont').disable(); + this.polarAreaChartWidgetConfigForm.get('legendLabelColor').disable(); + this.polarAreaChartWidgetConfigForm.get('legendValueFont').disable(); + this.polarAreaChartWidgetConfigForm.get('legendValueColor').disable(); + } + if (showTooltip) { + this.polarAreaChartWidgetConfigForm.get('tooltipValueType').enable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueDecimals').enable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueFont').enable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueColor').enable(); + this.polarAreaChartWidgetConfigForm.get('tooltipBackgroundColor').enable(); + this.polarAreaChartWidgetConfigForm.get('tooltipBackgroundBlur').enable(); + } else { + this.polarAreaChartWidgetConfigForm.get('tooltipValueType').disable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueDecimals').disable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueFont').disable(); + this.polarAreaChartWidgetConfigForm.get('tooltipValueColor').disable(); + this.polarAreaChartWidgetConfigForm.get('tooltipBackgroundColor').disable(); + this.polarAreaChartWidgetConfigForm.get('tooltipBackgroundBlur').disable(); + } + } + + private getSeries(datasources?: Datasource[]): DataKey[] { + if (datasources && datasources.length) { + return datasources[0].dataKeys || []; + } + return []; + } + + private setSeries(series: DataKey[], datasources?: Datasource[]) { + if (datasources && datasources.length) { + datasources[0].dataKeys = series; + } + } + + private getCardButtons(config: WidgetConfig): string[] { + const buttons: string[] = []; + if (isUndefined(config.enableFullscreen) || config.enableFullscreen) { + buttons.push('fullscreen'); + } + return buttons; + } + + private setCardButtons(buttons: string[], config: WidgetConfig) { + config.enableFullscreen = buttons.includes('fullscreen'); + } + + private _valuePreviewFn(): string { + const units: string = this.polarAreaChartWidgetConfigForm.get('units').value; + const decimals: number = this.polarAreaChartWidgetConfigForm.get('decimals').value; + return formatValue(110, decimals, units, false); + } + + private _tooltipValuePreviewFn(): string { + const tooltipValueType: LatestChartTooltipValueType = this.polarAreaChartWidgetConfigForm.get('tooltipValueType').value; + const decimals: number = this.polarAreaChartWidgetConfigForm.get('tooltipValueDecimals').value; + if (tooltipValueType === LatestChartTooltipValueType.percentage) { + return formatValue(35, decimals, '%', false); + } else { + const units: string = this.polarAreaChartWidgetConfigForm.get('units').value; + return formatValue(110, decimals, units, false); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.html index 913974c313..1ba564b3ba 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.html @@ -186,8 +186,8 @@
- - {{ seriesLabelPositionTranslations.get(position) | translate }} + + {{ chartLabelPositionTranslations.get(position) | translate }} @@ -216,8 +216,8 @@
widgets.time-series-chart.series.point.point-shape
- - {{ echartsShapeTranslations.get(shape) | translate }} + + {{ chartShapeTranslations.get(shape) | translate }} @@ -367,9 +367,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts index 1144f0b905..985fd74624 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/range-chart-basic-config.component.ts @@ -47,12 +47,10 @@ import { import { lineSeriesStepTypes, lineSeriesStepTypeTranslations, - seriesLabelPositions, - seriesLabelPositionTranslations, timeSeriesLineTypes, timeSeriesLineTypeTranslations } from '@home/components/widget/lib/chart/time-series-chart.models'; -import { echartsShapes, echartsShapeTranslations } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { chartLabelPositions, chartLabelPositionTranslations, chartShapes, chartShapeTranslations } from '@home/components/widget/lib/chart/chart.models'; @Component({ selector: 'tb-range-chart-basic-config', @@ -78,13 +76,13 @@ export class RangeChartBasicConfigComponent extends BasicWidgetConfigComponent { timeSeriesLineTypeTranslations = timeSeriesLineTypeTranslations; - seriesLabelPositions = seriesLabelPositions; + chartLabelPositions = chartLabelPositions; - seriesLabelPositionTranslations = seriesLabelPositionTranslations; + chartLabelPositionTranslations = chartLabelPositionTranslations; - echartsShapes = echartsShapes; + chartShapes = chartShapes; - echartsShapeTranslations = echartsShapeTranslations; + chartShapeTranslations = chartShapeTranslations; legendPositions = legendPositions; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.html index f8e72b5676..3d0fe04e4c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.html @@ -328,9 +328,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.ts index f32f35d47e..9a8cb6a7b3 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/chart/time-series-chart-basic-config.component.ts @@ -45,8 +45,8 @@ import { timeSeriesChartWidgetDefaultSettings, TimeSeriesChartWidgetSettings } from '@home/components/widget/lib/chart/time-series-chart-widget.models'; -import { EChartsTooltipTrigger } from '@home/components/widget/lib/chart/echarts-widget.models'; import { + TimeSeriesChartTooltipTrigger, TimeSeriesChartKeySettings, TimeSeriesChartType, TimeSeriesChartYAxes, @@ -76,7 +76,7 @@ export class TimeSeriesChartBasicConfigComponent extends BasicWidgetConfigCompon TimeSeriesChartType = TimeSeriesChartType; - EChartsTooltipTrigger = EChartsTooltipTrigger; + EChartsTooltipTrigger = TimeSeriesChartTooltipTrigger; legendPositions = legendPositions; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.component.ts new file mode 100644 index 0000000000..b39066cf8a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.component.ts @@ -0,0 +1,75 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, TemplateRef, ViewChild, ViewEncapsulation } from '@angular/core'; +import { WidgetContext } from '@home/models/widget-component.models'; +import { WidgetComponent } from '@home/components/widget/widget.component'; +import { TranslateService } from '@ngx-translate/core'; +import { + LatestChartComponent, + LatestChartComponentCallbacks +} from '@home/components/widget/lib/chart/latest-chart.component'; +import { + barChartWidgetBarsChartSettings, + barChartWidgetDefaultSettings, + BarChartWidgetSettings +} from '@home/components/widget/lib/chart/bar-chart-widget.models'; +import { TbBarsChart } from '@home/components/widget/lib/chart/bars-chart'; + +@Component({ + selector: 'tb-bar-chart-widget', + templateUrl: './latest-chart-widget.component.html', + styleUrls: [], + encapsulation: ViewEncapsulation.None +}) +export class BarChartWidgetComponent implements OnInit { + + @ViewChild('latestChart') + latestChart: LatestChartComponent; + + @Input() + ctx: WidgetContext; + + @Input() + widgetTitlePanel: TemplateRef; + + settings: BarChartWidgetSettings; + + callbacks: LatestChartComponentCallbacks; + + constructor(private widgetComponent: WidgetComponent, + private translate: TranslateService) { + } + + ngOnInit(): void { + this.ctx.$scope.barChartWidget = this; + this.settings = {...barChartWidgetDefaultSettings, ...this.ctx.settings}; + this.callbacks = { + createChart: (chartShape, renderer) => { + const settings = barChartWidgetBarsChartSettings(this.settings); + return new TbBarsChart(this.ctx, settings, chartShape.nativeElement, renderer, this.translate, true); + } + }; + } + + public onInit() { + this.latestChart?.onInit(); + } + + public onDataUpdated() { + this.latestChart?.onDataUpdated(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.models.ts new file mode 100644 index 0000000000..b51368b071 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-widget.models.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + latestChartWidgetDefaultSettings, + LatestChartWidgetSettings +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { mergeDeep } from '@core/utils'; +import { + barsChartAnimationDefaultSettings, + BarsChartSettings +} from '@home/components/widget/lib/chart/bars-chart.models'; +import { Font } from '@shared/models/widget-settings.models'; +import { DeepPartial } from '@shared/models/common'; +import { + ChartAnimationSettings, + chartBarDefaultSettings, + ChartBarSettings, + chartColorScheme +} from '@home/components/widget/lib/chart/chart.models'; + +export interface BarChartWidgetSettings extends LatestChartWidgetSettings { + axisMin?: number; + axisMax?: number; + axisTickLabelFont: Font; + axisTickLabelColor: string; + barSettings: ChartBarSettings; +} + +export const barChartWidgetDefaultSettings: BarChartWidgetSettings = { + ...latestChartWidgetDefaultSettings, + animation: mergeDeep({} as ChartAnimationSettings, + barsChartAnimationDefaultSettings), + axisTickLabelFont: { + family: 'Roboto', + size: 12, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '1' + }, + axisTickLabelColor: chartColorScheme['axis.tickLabel'].light, + barSettings: mergeDeep({} as ChartBarSettings, chartBarDefaultSettings, + {barWidth: 80, showLabel: true} as ChartBarSettings) +}; + +export const barChartWidgetBarsChartSettings = (settings: BarChartWidgetSettings): DeepPartial => ({ + polar: false, + axisMin: settings.axisMin, + axisMax: settings.axisMax, + axisTickLabelFont: settings.axisTickLabelFont, + axisTickLabelColor: settings.axisTickLabelColor, + barSettings: settings.barSettings, + sortSeries: settings.sortSeries, + showTotal: false, + animation: settings.animation, + showLegend: settings.showLegend, + showTooltip: settings.showTooltip, + tooltipValueType: settings.tooltipValueType, + tooltipValueDecimals: settings.tooltipValueDecimals, + tooltipValueFont: settings.tooltipValueFont, + tooltipValueColor: settings.tooltipValueColor, + tooltipBackgroundColor: settings.tooltipBackgroundColor, + tooltipBackgroundBlur: settings.tooltipBackgroundBlur +}); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.models.ts index 5351b219a3..185922a3eb 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bar-chart-with-labels-widget.models.ts @@ -23,17 +23,10 @@ import { textStyle } from '@shared/models/widget-settings.models'; import { LegendPosition } from '@shared/models/widget.models'; -import { - echartsAnimationDefaultSettings, - EChartsAnimationSettings, - EChartsTooltipWidgetSettings -} from '@home/components/widget/lib/chart/echarts-widget.models'; import { DeepPartial } from '@shared/models/common'; import { defaultTimeSeriesChartXAxisSettings, defaultTimeSeriesChartYAxisSettings, - SeriesFillSettings, - SeriesFillType, timeSeriesChartGridDefaultSettings, TimeSeriesChartGridSettings, TimeSeriesChartKeySettings, @@ -42,14 +35,21 @@ import { TimeSeriesChartSeriesType, TimeSeriesChartSettings, TimeSeriesChartThreshold, + TimeSeriesChartTooltipWidgetSettings, TimeSeriesChartXAxisSettings, TimeSeriesChartYAxisSettings } from '@home/components/widget/lib/chart/time-series-chart.models'; import { CallbackDataParams, LabelLayoutOptionCallbackParams } from 'echarts/types/dist/shared'; import { formatValue, mergeDeep } from '@core/utils'; import { LabelLayoutOption } from 'echarts/types/src/util/types'; +import { + chartAnimationDefaultSettings, + ChartAnimationSettings, + ChartFillSettings, + ChartFillType +} from '@home/components/widget/lib/chart/chart.models'; -export interface BarChartWithLabelsWidgetSettings extends EChartsTooltipWidgetSettings { +export interface BarChartWithLabelsWidgetSettings extends TimeSeriesChartTooltipWidgetSettings { dataZoom: boolean; showBarLabel: boolean; barLabelFont: Font; @@ -60,12 +60,12 @@ export interface BarChartWithLabelsWidgetSettings extends EChartsTooltipWidgetSe showBarBorder: boolean; barBorderWidth: number; barBorderRadius: number; - barBackgroundSettings: SeriesFillSettings; + barBackgroundSettings: ChartFillSettings; noAggregationBarWidthSettings: TimeSeriesChartNoAggregationBarWidthSettings; grid: TimeSeriesChartGridSettings; yAxis: TimeSeriesChartYAxisSettings; xAxis: TimeSeriesChartXAxisSettings; - animation: EChartsAnimationSettings; + animation: ChartAnimationSettings; thresholds: TimeSeriesChartThreshold[]; showLegend: boolean; legendPosition: LegendPosition; @@ -101,7 +101,7 @@ export const barChartWithLabelsDefaultSettings: BarChartWithLabelsWidgetSettings barBorderWidth: 2, barBorderRadius: 0, barBackgroundSettings: { - type: SeriesFillType.none, + type: ChartFillType.none, opacity: 0.4, gradient: { start: 100, @@ -118,8 +118,8 @@ export const barChartWithLabelsDefaultSettings: BarChartWithLabelsWidgetSettings xAxis: mergeDeep({} as TimeSeriesChartXAxisSettings, defaultTimeSeriesChartXAxisSettings, {showTicks: false, showSplitLines: false} as TimeSeriesChartXAxisSettings), - animation: mergeDeep({} as EChartsAnimationSettings, - echartsAnimationDefaultSettings), + animation: mergeDeep({} as ChartAnimationSettings, + chartAnimationDefaultSettings), thresholds: [], showLegend: true, legendPosition: LegendPosition.top, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.models.ts new file mode 100644 index 0000000000..a099d11f7d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.models.ts @@ -0,0 +1,61 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { latestChartDefaultSettings, LatestChartSettings } from '@home/components/widget/lib/chart/latest-chart.models'; +import { mergeDeep } from '@core/utils'; +import { + chartAnimationDefaultSettings, + ChartAnimationSettings, + chartBarDefaultSettings, + ChartBarSettings, + chartColorScheme +} from '@home/components/widget/lib/chart/chart.models'; +import { Font } from '@shared/models/widget-settings.models'; + +export interface BarsChartSettings extends LatestChartSettings { + polar: boolean; + axisMin?: number | string; + axisMax?: number | string; + axisTickLabelFont: Font; + axisTickLabelColor: string; + angleAxisStartAngle?: number; + barSettings: ChartBarSettings; +} + +export const barsChartAnimationDefaultSettings: ChartAnimationSettings = + mergeDeep({} as ChartAnimationSettings, chartAnimationDefaultSettings, { + animationDuration: 1000, + animationDurationUpdate: 500 + } as ChartAnimationSettings); + +export const barsChartDefaultSettings: BarsChartSettings = { + ...latestChartDefaultSettings, + animation: mergeDeep({} as ChartAnimationSettings, + barsChartAnimationDefaultSettings), + polar: false, + axisTickLabelFont: { + family: 'Roboto', + size: 12, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '1' + }, + axisTickLabelColor: chartColorScheme['axis.tickLabel'].light, + angleAxisStartAngle: 0, + barSettings: mergeDeep({} as ChartBarSettings, chartBarDefaultSettings, + {barWidth: 80, showLabel: true} as ChartBarSettings) +}; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.ts new file mode 100644 index 0000000000..f661223983 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/bars-chart.ts @@ -0,0 +1,173 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { TbLatestChart } from '@home/components/widget/lib/chart/latest-chart'; +import { barsChartDefaultSettings, BarsChartSettings } from '@home/components/widget/lib/chart/bars-chart.models'; +import { WidgetContext } from '@home/models/widget-component.models'; +import { DeepPartial } from '@shared/models/common'; +import { PieChartSettings } from '@home/components/widget/lib/chart/pie-chart.models'; +import { Renderer2 } from '@angular/core'; +import { TranslateService } from '@ngx-translate/core'; +import { ComponentStyle } from '@shared/models/widget-settings.models'; +import { LinearGradientObject } from 'zrender/lib/graphic/LinearGradient'; +import tinycolor from 'tinycolor2'; +import { BarDataItemOption, BarSeriesLabelOption } from 'echarts/types/src/chart/bar/BarSeries'; +import { formatValue, isDefinedAndNotNull } from '@core/utils'; +import { + ChartFillType, + ChartLabelPosition, + createChartTextStyle, + createLinearOpacityGradient +} from '@home/components/widget/lib/chart/chart.models'; +import { ValueAxisBaseOption } from 'echarts/types/src/coord/axisCommonTypes'; +import { RadiusAxisOption, YAXisOption } from 'echarts/types/dist/shared'; + +export class TbBarsChart extends TbLatestChart { + + constructor(ctx: WidgetContext, + inputSettings: DeepPartial, + chartElement: HTMLElement, + renderer: Renderer2, + translate: TranslateService, + autoResize = true) { + + super(ctx, inputSettings, chartElement, renderer, translate, autoResize); + } + + protected defaultSettings(): BarsChartSettings { + return barsChartDefaultSettings; + } + + protected prepareLatestChartOption() { + let labelStyle: ComponentStyle = {}; + if (this.settings.barSettings.showLabel) { + labelStyle = createChartTextStyle(this.settings.barSettings.labelFont, + this.settings.barSettings.labelColor, false, 'series.label', false); + } + const labelOption: BarSeriesLabelOption = { + show: this.settings.barSettings.showLabel, + position: this.settings.barSettings.labelPosition, + formatter: (params) => `{label|${params.name}}`, + rich: { + label: labelStyle + } + }; + if (this.settings.barSettings.enableLabelBackground) { + labelOption.backgroundColor = this.settings.barSettings.labelBackground; + labelOption.padding = [4, 5]; + labelOption.borderRadius = 4; + } + this.latestChartOption.series = [ + { + type: 'bar', + barWidth: isDefinedAndNotNull(this.settings.barSettings.barWidth) ? this.settings.barSettings.barWidth + '%' : undefined, + itemStyle: { + borderWidth: this.settings.barSettings.showBorder ? this.settings.barSettings.borderWidth : 0 + }, + emphasis: { + focus: 'self' + }, + coordinateSystem: this.settings.polar ? 'polar' : 'cartesian2d', + label: labelOption, + animation: this.settings.animation.animation, + animationThreshold: this.settings.animation.animationThreshold, + animationDuration: this.settings.animation.animationDuration, + animationEasing: this.settings.animation.animationEasing, + animationDelay: this.settings.animation.animationDelay, + animationDurationUpdate: this.settings.animation.animationDurationUpdate, + animationEasingUpdate: this.settings.animation.animationEasingUpdate, + animationDelayUpdate: this.settings.animation.animationDelayUpdate + } + ]; + + const axisTickLabelStyle = createChartTextStyle(this.settings.axisTickLabelFont, + this.settings.axisTickLabelColor, false, 'axis.tickLabel'); + const valueAxis: ValueAxisBaseOption = { + type: 'value', + min: this.settings.axisMin, + max: this.settings.axisMax, + axisLabel: { + color: axisTickLabelStyle.color, + fontStyle: axisTickLabelStyle.fontStyle, + fontWeight: axisTickLabelStyle.fontWeight, + fontFamily: axisTickLabelStyle.fontFamily, + fontSize: axisTickLabelStyle.fontSize, + formatter: (value: any) => formatValue(value, this.decimals, this.units, false) + } + }; + if (this.settings.polar) { + this.latestChartOption.polar = { + radius: '100%' + }; + this.latestChartOption.radiusAxis = valueAxis as RadiusAxisOption; + this.latestChartOption.angleAxis = { + type: 'category', + data: [], + startAngle: this.settings.angleAxisStartAngle + }; + } else { + let minTop = 0; + let minBottom = 0; + if (this.settings.barSettings.showLabel) { + if (this.settings.barSettings.labelPosition === ChartLabelPosition.top) { + minTop = this.settings.barSettings.labelFont.size; + } else if (this.settings.barSettings.labelPosition === ChartLabelPosition.bottom) { + minBottom = this.settings.barSettings.labelFont.size; + } + } + this.latestChartOption.grid = [{ + containLabel: true, + top: minTop, + bottom: minBottom, + left: 0, + right: 0 + }]; + this.latestChartOption.xAxis = { + type: 'category', + data: [] + }; + this.latestChartOption.yAxis = valueAxis as YAXisOption; + } + } + + protected doUpdateSeriesData() { + const seriesData: BarDataItemOption[] = []; + for (const dataItem of this.dataItems) { + if (dataItem.enabled && dataItem.hasValue) { + const barSettings = this.settings.barSettings; + let borderRadius: number[]; + if (dataItem.value < 0) { + borderRadius = [0, 0, barSettings.borderRadius, barSettings.borderRadius]; + } else { + borderRadius = [barSettings.borderRadius, barSettings.borderRadius, 0, 0]; + } + let barColor: string | LinearGradientObject; + if (barSettings.backgroundSettings.type === ChartFillType.none) { + barColor = dataItem.dataKey.color; + } else if (barSettings.backgroundSettings.type === ChartFillType.opacity) { + barColor = tinycolor(dataItem.dataKey.color).setAlpha(barSettings.backgroundSettings.opacity).toRgbString(); + } else { + barColor = createLinearOpacityGradient(dataItem.dataKey.color, barSettings.backgroundSettings.gradient); + } + seriesData.push( + {id: dataItem.id, value: dataItem.value, name: dataItem.dataKey.label, + itemStyle: {color: barColor, borderColor: dataItem.dataKey.color, borderRadius}} + ); + } + } + this.latestChartOption.series[0].data = seriesData; + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/chart.models.ts new file mode 100644 index 0000000000..1f81f4bc3e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/chart.models.ts @@ -0,0 +1,316 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { isNumber } from '@core/utils'; +import { TbColorScheme } from '@shared/models/color.models'; +import { LinearGradientObject } from 'zrender/lib/graphic/LinearGradient'; +import tinycolor from 'tinycolor2'; +import { ComponentStyle, Font, textStyle } from '@shared/models/widget-settings.models'; +import { LabelFormatterCallback } from 'echarts'; +import { LabelLayoutOption } from 'echarts/types/src/util/types'; +import { LabelLayoutOptionCallback } from 'echarts/types/dist/shared'; +import { BuiltinTextPosition } from 'zrender/src/core/types'; + +export const chartColorScheme: TbColorScheme = { + 'threshold.line': { + light: 'rgba(0, 0, 0, 0.76)', + dark: '#eee' + }, + 'threshold.label': { + light: 'rgba(0, 0, 0, 0.76)', + dark: '#eee' + }, + 'axis.line': { + light: 'rgba(0, 0, 0, 0.54)', + dark: '#B9B8CE' + }, + 'axis.label': { + light: 'rgba(0, 0, 0, 0.54)', + dark: '#B9B8CE' + }, + 'axis.ticks': { + light: 'rgba(0, 0, 0, 0.54)', + dark: '#B9B8CE' + }, + 'axis.tickLabel': { + light: 'rgba(0, 0, 0, 0.54)', + dark: '#B9B8CE' + }, + 'axis.splitLine': { + light: 'rgba(0, 0, 0, 0.12)', + dark: '#484753' + }, + 'series.label': { + light: 'rgba(0, 0, 0, 0.76)', + dark: '#eee' + } +}; + +export enum ChartShape { + emptyCircle = 'emptyCircle', + circle = 'circle', + rect = 'rect', + roundRect = 'roundRect', + triangle = 'triangle', + diamond = 'diamond', + pin = 'pin', + arrow = 'arrow', + none = 'none' +} + +export const chartShapes = Object.keys(ChartShape) as ChartShape[]; + +export const chartShapeTranslations = new Map( + [ + [ChartShape.emptyCircle, 'widgets.chart.shape-empty-circle'], + [ChartShape.circle, 'widgets.chart.shape-circle'], + [ChartShape.rect, 'widgets.chart.shape-rect'], + [ChartShape.roundRect, 'widgets.chart.shape-round-rect'], + [ChartShape.triangle, 'widgets.chart.shape-triangle'], + [ChartShape.diamond, 'widgets.chart.shape-diamond'], + [ChartShape.pin, 'widgets.chart.shape-pin'], + [ChartShape.arrow, 'widgets.chart.shape-arrow'], + [ChartShape.none, 'widgets.chart.shape-none'] + ] +); + +export enum ChartAnimationEasing { + linear = 'linear', + quadraticIn = 'quadraticIn', + quadraticOut = 'quadraticOut', + quadraticInOut = 'quadraticInOut', + cubicIn = 'cubicIn', + cubicOut = 'cubicOut', + cubicInOut = 'cubicInOut', + quarticIn = 'quarticIn', + quarticOut = 'quarticOut', + quarticInOut = 'quarticInOut', + quinticIn = 'quinticIn', + quinticOut = 'quinticOut', + quinticInOut = 'quinticInOut', + sinusoidalIn = 'sinusoidalIn', + sinusoidalOut = 'sinusoidalOut', + sinusoidalInOut = 'sinusoidalInOut', + exponentialIn = 'exponentialIn', + exponentialOut = 'exponentialOut', + exponentialInOut = 'exponentialInOut', + circularIn = 'circularIn', + circularOut = 'circularOut', + circularInOut = 'circularInOut', + elasticIn = 'elasticIn', + elasticOut = 'elasticOut', + elasticInOut = 'elasticInOut', + backIn = 'backIn', + backOut = 'backOut', + backInOut = 'backInOut', + bounceIn = 'bounceIn', + bounceOut = 'bounceOut', + bounceInOut = 'bounceInOut' +} + +export const chartAnimationEasings = Object.keys(ChartAnimationEasing) as ChartAnimationEasing[]; + +export enum ChartFillType { + none = 'none', + opacity = 'opacity', + gradient = 'gradient' +} + +export const chartFillTypes = Object.keys(ChartFillType) as ChartFillType[]; + +export const chartFillTypeTranslations = new Map( + [ + [ChartFillType.none, 'widgets.chart.fill-type-none'], + [ChartFillType.opacity, 'widgets.chart.fill-type-opacity'], + [ChartFillType.gradient, 'widgets.chart.fill-type-gradient'] + ] +); + +export enum ChartLabelPosition { + top = 'top', + bottom = 'bottom' +} + +export const chartLabelPositions = Object.keys(ChartLabelPosition) as ChartLabelPosition[]; + +export const chartLabelPositionTranslations = new Map( + [ + [ChartLabelPosition.top, 'widgets.chart.label-position-top'], + [ChartLabelPosition.bottom, 'widgets.chart.label-position-bottom'] + ] +); + +export enum PieChartLabelPosition { + inside = 'inside', + outside = 'outside' +} + +export const pieChartLabelPositions = Object.keys(PieChartLabelPosition) as PieChartLabelPosition[]; + +export const pieChartLabelPositionTranslations = new Map( + [ + [PieChartLabelPosition.inside, 'widgets.chart.label-position-inside'], + [PieChartLabelPosition.outside, 'widgets.chart.label-position-outside'] + ] +); + +export interface ChartAnimationSettings { + animation: boolean; + animationThreshold: number; + animationDuration: number; + animationEasing: ChartAnimationEasing; + animationDelay: number; + animationDurationUpdate: number; + animationEasingUpdate: ChartAnimationEasing; + animationDelayUpdate: number; +} + +export const chartAnimationDefaultSettings: ChartAnimationSettings = { + animation: true, + animationThreshold: 2000, + animationDuration: 500, + animationEasing: ChartAnimationEasing.cubicOut, + animationDelay: 0, + animationDurationUpdate: 300, + animationEasingUpdate: ChartAnimationEasing.cubicOut, + animationDelayUpdate: 0 +}; + +export interface ChartFillSettings { + type: ChartFillType; + opacity: number; + gradient: { + start: number; + end: number; + }; +} + +export interface ChartBarSettings { + showBorder: boolean; + borderWidth: number; + borderRadius: number; + barWidth?: number; + showLabel: boolean; + labelPosition: ChartLabelPosition | PieChartLabelPosition | BuiltinTextPosition; + labelFont: Font; + labelColor: string; + enableLabelBackground: boolean; + labelBackground: string; + labelFormatter?: string | LabelFormatterCallback; + labelLayout?: LabelLayoutOption | LabelLayoutOptionCallback; + additionalLabelOption?: {[key: string]: any}; + backgroundSettings: ChartFillSettings; +} + +export const chartBarDefaultSettings: ChartBarSettings = { + showBorder: false, + borderWidth: 2, + borderRadius: 0, + showLabel: false, + labelPosition: ChartLabelPosition.top, + labelFont: { + family: 'Roboto', + size: 11, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '1' + }, + labelColor: chartColorScheme['series.label'].light, + enableLabelBackground: false, + labelBackground: 'rgba(255,255,255,0.56)', + backgroundSettings: { + type: ChartFillType.none, + opacity: 0.4, + gradient: { + start: 100, + end: 0 + } + } +}; + +type ChartShapeOffsetFunction = (size: number) => number; + +const chartShapeOffsetFunctions = new Map( + [ + [ChartShape.emptyCircle, size => size / 2 + 1], + [ChartShape.circle, size => size / 2], + [ChartShape.rect, size => size / 2], + [ChartShape.roundRect, size => size / 2], + [ChartShape.triangle, size => size / 2], + [ChartShape.diamond, size => size / 2], + [ChartShape.pin, size => size], + [ChartShape.arrow, () => 0], + [ChartShape.none, () => 0], + ] +); + +export const measureSymbolOffset = (symbol: string, symbolSize: any): number => { + if (isNumber(symbolSize)) { + if (symbol) { + const offsetFunction = chartShapeOffsetFunctions.get(symbol as ChartShape); + if (offsetFunction) { + return offsetFunction(symbolSize); + } else { + return symbolSize / 2; + } + } + } else { + return 0; + } +}; + +export const createLinearOpacityGradient = (color: string, gradient: {start: number; end: number}): LinearGradientObject => ({ + type: 'linear', + x: 0, + y: 0, + x2: 0, + y2: 1, + colorStops: [{ + offset: 0, color: tinycolor(color).setAlpha(gradient.start / 100).toRgbString() // color at 0% + }, { + offset: 1, color: tinycolor(color).setAlpha(gradient.end / 100).toRgbString() // color at 100% + }], + global: false +}); + +export const createChartTextStyle = (font: Font, color: string, darkMode: boolean, colorKey?: string, fill = false): ComponentStyle => { + const style = textStyle(font); + delete style.lineHeight; + style.fontSize = font.size; + if (fill) { + style.fill = prepareChartThemeColor(color, darkMode, colorKey); + } else { + style.color = prepareChartThemeColor(color, darkMode, colorKey); + } + return style; +}; + +export const prepareChartThemeColor = (color: string, darkMode: boolean, colorKey?: string): string => { + if (darkMode) { + let colorInstance = tinycolor(color); + if (colorInstance.isDark()) { + if (colorKey && chartColorScheme[colorKey]) { + return chartColorScheme[colorKey].dark; + } else { + const rgb = colorInstance.toRgb(); + colorInstance = tinycolor({r: 255 - rgb.r, g: 255 - rgb.g, b: 255 - rgb.b, a: rgb.a}); + return colorInstance.toRgbString(); + } + } + } + return color; +}; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/doughnut-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/doughnut-widget.models.ts index d059e0186c..c8c6f724e5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/doughnut-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/doughnut-widget.models.ts @@ -14,16 +14,16 @@ /// limitations under the License. /// -import { BackgroundType, ColorSettings, constantColor, Font } from '@shared/models/widget-settings.models'; +import { ColorSettings, constantColor, Font } from '@shared/models/widget-settings.models'; import { LegendPosition } from '@shared/models/widget.models'; import { pieChartAnimationDefaultSettings, PieChartSettings } from '@home/components/widget/lib/chart/pie-chart.models'; import { DeepPartial } from '@shared/models/common'; import { - LatestChartTooltipValueType, + latestChartWidgetDefaultSettings, LatestChartWidgetSettings } from '@home/components/widget/lib/chart/latest-chart.models'; import { mergeDeep } from '@core/utils'; -import { EChartsAnimationSettings } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { ChartAnimationSettings } from '@home/components/widget/lib/chart/chart.models'; export enum DoughnutLayout { default = 'default', @@ -61,10 +61,14 @@ export interface DoughnutWidgetSettings extends LatestChartWidgetSettings { } export const doughnutDefaultSettings = (horizontal: boolean): DoughnutWidgetSettings => ({ - layout: DoughnutLayout.default, + ...latestChartWidgetDefaultSettings, autoScale: true, - clockwise: false, sortSeries: false, + animation: mergeDeep({} as ChartAnimationSettings, + pieChartAnimationDefaultSettings), + legendPosition: horizontal ? LegendPosition.right : LegendPosition.bottom, + layout: DoughnutLayout.default, + clockwise: false, totalValueFont: { family: 'Roboto', size: 24, @@ -73,52 +77,7 @@ export const doughnutDefaultSettings = (horizontal: boolean): DoughnutWidgetSett weight: '500', lineHeight: '1' }, - totalValueColor: constantColor('rgba(0, 0, 0, 0.87)'), - animation: mergeDeep({} as EChartsAnimationSettings, - pieChartAnimationDefaultSettings), - showLegend: true, - legendPosition: horizontal ? LegendPosition.right : LegendPosition.bottom, - legendLabelFont: { - family: 'Roboto', - size: 12, - sizeUnit: 'px', - style: 'normal', - weight: '400', - lineHeight: '16px' - }, - legendLabelColor: 'rgba(0, 0, 0, 0.38)', - legendValueFont: { - family: 'Roboto', - size: 14, - sizeUnit: 'px', - style: 'normal', - weight: '500', - lineHeight: '20px' - }, - legendValueColor: 'rgba(0, 0, 0, 0.87)', - showTooltip: true, - tooltipValueType: LatestChartTooltipValueType.percentage, - tooltipValueDecimals: 0, - tooltipValueFont: { - family: 'Roboto', - size: 13, - sizeUnit: 'px', - style: 'normal', - weight: '500', - lineHeight: '16px' - }, - tooltipValueColor: 'rgba(0, 0, 0, 0.76)', - tooltipBackgroundColor: 'rgba(255, 255, 255, 0.76)', - tooltipBackgroundBlur: 4, - background: { - type: BackgroundType.color, - color: '#fff', - overlay: { - enabled: false, - color: 'rgba(255,255,255,0.72)', - blur: 3 - } - } + totalValueColor: constantColor('rgba(0, 0, 0, 0.87)') }); export const doughnutPieChartSettings = (settings: DoughnutWidgetSettings): DeepPartial => ({ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts index a225973b82..121d46c86d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/echarts-widget.models.ts @@ -17,7 +17,6 @@ import * as echarts from 'echarts/core'; import AxisModel from 'echarts/types/src/coord/cartesian/AxisModel'; import { estimateLabelUnionRect } from 'echarts/lib/coord/axisHelper'; -import { isDefinedAndNotNull, isFunction, isNumber } from '@core/utils'; import { DataZoomComponent, DataZoomComponentOption, @@ -25,6 +24,8 @@ import { GridComponentOption, MarkLineComponent, MarkLineComponentOption, + PolarComponent, + PolarComponentOption, TooltipComponent, TooltipComponentOption, VisualMapComponent, @@ -42,21 +43,12 @@ import { } from 'echarts/charts'; import { LabelLayout } from 'echarts/features'; import { CanvasRenderer, SVGRenderer } from 'echarts/renderers'; -import { DataEntry, DataKey, DataSet, Datasource, FormattedData } from '@shared/models/widget.models'; -import { - calculateAggIntervalWithWidgetTimeWindow, - Interval, - IntervalMath, - WidgetTimewindow -} from '@shared/models/time/time.models'; -import { CallbackDataParams, TimeAxisBandWidthCalculator } from 'echarts/types/dist/shared'; -import { Renderer2 } from '@angular/core'; -import { DateFormatProcessor, DateFormatSettings, Font } from '@shared/models/widget-settings.models'; +import { CallbackDataParams } from 'echarts/types/dist/shared'; import GlobalModel from 'echarts/types/src/model/Global'; import Axis2D from 'echarts/types/src/coord/cartesian/Axis2D'; import SeriesModel from 'echarts/types/src/model/Series'; import { MarkLine2DDataItemOption } from 'echarts/types/src/component/marker/MarkLineModel'; -import { TimeAxisBaseOption } from 'echarts/types/src/coord/axisCommonTypes'; +import { measureSymbolOffset } from '@home/components/widget/lib/chart/chart.models'; class EChartsModule { private initialized = false; @@ -69,6 +61,7 @@ class EChartsModule { VisualMapComponent, DataZoomComponent, MarkLineComponent, + PolarComponent, LineChart, BarChart, PieChart, @@ -90,6 +83,7 @@ export type EChartsOption = echarts.ComposeOption< | VisualMapComponentOption | DataZoomComponentOption | MarkLineComponentOption + | PolarComponentOption | LineSeriesOption | CustomSeriesOption | BarSeriesOption @@ -98,166 +92,6 @@ export type EChartsOption = echarts.ComposeOption< export type ECharts = echarts.ECharts; -export type EChartsDataItem = [number, any, number, number]; - -export type NamedDataSet = {name: string; value: EChartsDataItem}[]; - -export type EChartsTooltipValueFormatFunction = (value: any, latestData: FormattedData, units?: string, decimals?: number) => string; - -export type EChartsSeriesItem = { - id: string; - datasource: Datasource; - dataKey: DataKey; - data: NamedDataSet; - dataSet?: DataSet; - enabled: boolean; - units?: string; - decimals?: number; - latestData?: FormattedData; - tooltipValueFormatFunction?: EChartsTooltipValueFormatFunction; - comparisonItem?: boolean; -}; - -export enum EChartsShape { - emptyCircle = 'emptyCircle', - circle = 'circle', - rect = 'rect', - roundRect = 'roundRect', - triangle = 'triangle', - diamond = 'diamond', - pin = 'pin', - arrow = 'arrow', - none = 'none' -} - -export const echartsShapes = Object.keys(EChartsShape) as EChartsShape[]; - -export const echartsShapeTranslations = new Map( - [ - [EChartsShape.emptyCircle, 'widgets.time-series-chart.shape-empty-circle'], - [EChartsShape.circle, 'widgets.time-series-chart.shape-circle'], - [EChartsShape.rect, 'widgets.time-series-chart.shape-rect'], - [EChartsShape.roundRect, 'widgets.time-series-chart.shape-round-rect'], - [EChartsShape.triangle, 'widgets.time-series-chart.shape-triangle'], - [EChartsShape.diamond, 'widgets.time-series-chart.shape-diamond'], - [EChartsShape.pin, 'widgets.time-series-chart.shape-pin'], - [EChartsShape.arrow, 'widgets.time-series-chart.shape-arrow'], - [EChartsShape.none, 'widgets.time-series-chart.shape-none'] - ] -); - -type EChartsShapeOffsetFunction = (size: number) => number; - -export const timeSeriesChartShapeOffsetFunctions = new Map( - [ - [EChartsShape.emptyCircle, size => size / 2 + 1], - [EChartsShape.circle, size => size / 2], - [EChartsShape.rect, size => size / 2], - [EChartsShape.roundRect, size => size / 2], - [EChartsShape.triangle, size => size / 2], - [EChartsShape.diamond, size => size / 2], - [EChartsShape.pin, size => size], - [EChartsShape.arrow, () => 0], - [EChartsShape.none, () => 0], - ] -); - -export enum EChartsAnimationEasing { - linear = 'linear', - quadraticIn = 'quadraticIn', - quadraticOut = 'quadraticOut', - quadraticInOut = 'quadraticInOut', - cubicIn = 'cubicIn', - cubicOut = 'cubicOut', - cubicInOut = 'cubicInOut', - quarticIn = 'quarticIn', - quarticOut = 'quarticOut', - quarticInOut = 'quarticInOut', - quinticIn = 'quinticIn', - quinticOut = 'quinticOut', - quinticInOut = 'quinticInOut', - sinusoidalIn = 'sinusoidalIn', - sinusoidalOut = 'sinusoidalOut', - sinusoidalInOut = 'sinusoidalInOut', - exponentialIn = 'exponentialIn', - exponentialOut = 'exponentialOut', - exponentialInOut = 'exponentialInOut', - circularIn = 'circularIn', - circularOut = 'circularOut', - circularInOut = 'circularInOut', - elasticIn = 'elasticIn', - elasticOut = 'elasticOut', - elasticInOut = 'elasticInOut', - backIn = 'backIn', - backOut = 'backOut', - backInOut = 'backInOut', - bounceIn = 'bounceIn', - bounceOut = 'bounceOut', - bounceInOut = 'bounceInOut' -} - -export const echartsAnimationEasings = Object.keys(EChartsAnimationEasing) as EChartsAnimationEasing[]; - -export interface EChartsAnimationSettings { - animation: boolean; - animationThreshold: number; - animationDuration: number; - animationEasing: EChartsAnimationEasing; - animationDelay: number; - animationDurationUpdate: number; - animationEasingUpdate: EChartsAnimationEasing; - animationDelayUpdate: number; -} - -export const echartsAnimationDefaultSettings: EChartsAnimationSettings = { - animation: true, - animationThreshold: 2000, - animationDuration: 500, - animationEasing: EChartsAnimationEasing.cubicOut, - animationDelay: 0, - animationDurationUpdate: 300, - animationEasingUpdate: EChartsAnimationEasing.cubicOut, - animationDelayUpdate: 0 -}; - -export const timeAxisBandWidthCalculator: TimeAxisBandWidthCalculator = (model) => { - let interval: number; - const axisOption = model.option; - const seriesDataIndices = axisOption.axisPointer?.seriesDataIndices; - if (seriesDataIndices?.length) { - const seriesDataIndex = seriesDataIndices[0]; - const series = model.ecModel.getSeriesByIndex(seriesDataIndex.seriesIndex); - if (series) { - const values = series.getData().getValues(seriesDataIndex.dataIndex); - const start = values[2]; - const end = values[3]; - if (typeof start === 'number' && typeof end === 'number') { - interval = Math.max(end - start, 1); - } - } - } - if (!interval) { - const tbTimeWindow: WidgetTimewindow = (axisOption as any).tbTimeWindow; - if (isDefinedAndNotNull(tbTimeWindow)) { - if (axisOption.axisPointer?.value && typeof axisOption.axisPointer?.value === 'number') { - const intervalArray = calculateAggIntervalWithWidgetTimeWindow(tbTimeWindow, axisOption.axisPointer.value); - const start = intervalArray[0]; - const end = intervalArray[1]; - interval = Math.max(end - start, 1); - } else { - interval = IntervalMath.numberValue(tbTimeWindow.interval); - } - } - } - if (interval) { - const timeScale = model.axis.scale; - const axisExtent = model.axis.getExtent(); - const dataExtent = timeScale.getExtent(); - const size = Math.abs(axisExtent[1] - axisExtent[0]); - return interval * (size / (dataExtent[1] - dataExtent[0])); - } -}; - export const getAxis = (chart: ECharts, mainType: string, axisId: string): Axis2D => { const model: GlobalModel = (chart as any).getModel(); const models = model.queryComponents({mainType, id: axisId}); @@ -298,21 +132,6 @@ const _calculateAxisSize = (axis: Axis2D): number => { return size; }; -const measureSymbolOffset = (symbol: string, symbolSize: any): number => { - if (isNumber(symbolSize)) { - if (symbol) { - const offsetFunction = timeSeriesChartShapeOffsetFunctions.get(symbol as EChartsShape); - if (offsetFunction) { - return offsetFunction(symbolSize); - } else { - return symbolSize / 2; - } - } - } else { - return 0; - } -}; - export const measureThresholdOffset = (chart: ECharts, axisId: string, thresholdId: string, value: any): [number, number] => { const offset: [number, number] = [0,0]; const axis = getAxis(chart, 'yAxis', axisId); @@ -435,294 +254,3 @@ export const getFocusedSeriesIndex = (chart: ECharts): number => { } return -1; }; - -export const toNamedData = (data: DataSet, valueConverter?: (value: any) => any): NamedDataSet => { - if (!data?.length) { - return []; - } else { - return data.map(d => { - const ts = isDefinedAndNotNull(d[2]) ? d[2][0] : d[0]; - return { - name: ts + '', - value: toEChartsDataItem(d, valueConverter) - }; - }); - } -}; - -const minDataTs = (dataSet: NamedDataSet): number => dataSet.length ? dataSet.map(data => - Number(data.name)).reduce((a, b) => Math.min(a, b)) : undefined; - -const maxDataTs = (dataSet: NamedDataSet): number => dataSet.length ? dataSet.map(data => - Number(data.name)).reduce((a, b) => Math.max(a, b)) : undefined; - -export const adjustTimeAxisExtentToData = (timeAxisOption: TimeAxisBaseOption, - dataItems: EChartsSeriesItem[], - defaultMin: number, - defaultMax: number): void => { - let min: number; - let max: number; - for (const item of dataItems) { - if (item.enabled) { - const minTs = minDataTs(item.data); - if (typeof minTs !== 'undefined') { - min = (typeof min !== 'undefined') ? Math.min(min, minTs) : minTs; - } - const maxTs = maxDataTs(item.data); - if (typeof maxTs !== 'undefined') { - max = (typeof max !== 'undefined') ? Math.max(max, maxTs) : maxTs; - } - } - } - timeAxisOption.min = (typeof min !== 'undefined' && Math.abs(min - defaultMin) < 1000) ? min : defaultMin; - timeAxisOption.max = (typeof max !== 'undefined' && Math.abs(max - defaultMax) < 1000) ? max : defaultMax; -}; - -const toEChartsDataItem = (entry: DataEntry, valueConverter?: (value: any) => any): EChartsDataItem => { - const value = valueConverter ? valueConverter(entry[1]) : entry[1]; - const item: EChartsDataItem = [entry[0], value, entry[0], entry[0]]; - if (isDefinedAndNotNull(entry[2])) { - item[2] = entry[2][0]; - item[3] = entry[2][1]; - } - return item; -}; - -export enum EChartsTooltipTrigger { - point = 'point', - axis = 'axis' -} - -export const tooltipTriggerTranslationMap = new Map( - [ - [ EChartsTooltipTrigger.point, 'tooltip.trigger-point' ], - [ EChartsTooltipTrigger.axis, 'tooltip.trigger-axis' ] - ] -); - -export interface EChartsTooltipWidgetSettings { - showTooltip: boolean; - tooltipTrigger?: EChartsTooltipTrigger; - tooltipShowFocusedSeries?: boolean; - tooltipLabelFont: Font; - tooltipLabelColor: string; - tooltipValueFont: Font; - tooltipValueColor: string; - tooltipValueFormatter?: string | EChartsTooltipValueFormatFunction; - tooltipShowDate: boolean; - tooltipDateInterval?: boolean; - tooltipDateFormat: DateFormatSettings; - tooltipDateFont: Font; - tooltipDateColor: string; - tooltipBackgroundColor: string; - tooltipBackgroundBlur: number; -} - -export const createTooltipValueFormatFunction = - (tooltipValueFormatter: string | EChartsTooltipValueFormatFunction): EChartsTooltipValueFormatFunction => { - let tooltipValueFormatFunction: EChartsTooltipValueFormatFunction; - if (isFunction(tooltipValueFormatter)) { - tooltipValueFormatFunction = tooltipValueFormatter as EChartsTooltipValueFormatFunction; - } else if (typeof tooltipValueFormatter === 'string' && tooltipValueFormatter.length) { - try { - tooltipValueFormatFunction = - new Function('value', 'latestData', tooltipValueFormatter) as EChartsTooltipValueFormatFunction; - } catch (e) {} - } - return tooltipValueFormatFunction; -}; - -export const echartsTooltipFormatter = (renderer: Renderer2, - tooltipDateFormat: DateFormatProcessor, - settings: EChartsTooltipWidgetSettings, - params: CallbackDataParams[] | CallbackDataParams, - valueFormatFunction: EChartsTooltipValueFormatFunction, - focusedSeriesIndex: number, - series?: EChartsSeriesItem[], - interval?: Interval): null | HTMLElement => { - - const tooltipParams = mapTooltipParams(params, series, focusedSeriesIndex); - if (!tooltipParams.items.length && !tooltipParams.comparisonItems.length) { - return null; - } - - const tooltipElement: HTMLElement = renderer.createElement('div'); - renderer.setStyle(tooltipElement, 'display', 'flex'); - renderer.setStyle(tooltipElement, 'flex-direction', 'column'); - renderer.setStyle(tooltipElement, 'align-items', 'flex-start'); - renderer.setStyle(tooltipElement, 'gap', '16px'); - - buildItemsTooltip(tooltipElement, tooltipParams.items, renderer, tooltipDateFormat, settings, valueFormatFunction, interval); - buildItemsTooltip(tooltipElement, tooltipParams.comparisonItems, renderer, tooltipDateFormat, settings, valueFormatFunction, interval); - - return tooltipElement; -}; - -interface TooltipItem { - param: CallbackDataParams; - dataItem: EChartsSeriesItem; -} - -interface TooltipParams { - items: TooltipItem[]; - comparisonItems: TooltipItem[]; -} - -const buildItemsTooltip = (tooltipElement: HTMLElement, - items: TooltipItem[], - renderer: Renderer2, - tooltipDateFormat: DateFormatProcessor, - settings: EChartsTooltipWidgetSettings, - valueFormatFunction: EChartsTooltipValueFormatFunction, - interval?: Interval) => { - if (items.length) { - const tooltipItemsElement: HTMLElement = renderer.createElement('div'); - renderer.setStyle(tooltipItemsElement, 'display', 'flex'); - renderer.setStyle(tooltipItemsElement, 'flex-direction', 'column'); - renderer.setStyle(tooltipItemsElement, 'align-items', 'flex-start'); - renderer.setStyle(tooltipItemsElement, 'gap', '4px'); - renderer.appendChild(tooltipElement, tooltipItemsElement); - if (settings.tooltipShowDate) { - renderer.appendChild(tooltipItemsElement, - constructEchartsTooltipDateElement(renderer, tooltipDateFormat, settings, items[0].param, interval)); - } - for (const item of items) { - renderer.appendChild(tooltipItemsElement, - constructEchartsTooltipSeriesElement(renderer, settings, item, valueFormatFunction)); - } - } -}; - -const mapTooltipParams = (params: CallbackDataParams[] | CallbackDataParams, - series?: EChartsSeriesItem[], - focusedSeriesIndex?: number): TooltipParams => { - const result: TooltipParams = { - items: [], - comparisonItems: [] - }; - if (!params || Array.isArray(params) && !params[0]) { - return result; - } - const firstParam = Array.isArray(params) ? params[0] : params; - if (!firstParam.value) { - return result; - } - let seriesParams: CallbackDataParams = null; - if (Array.isArray(params) && focusedSeriesIndex > -1) { - seriesParams = params.find(param => param.seriesIndex === focusedSeriesIndex); - } else if (!Array.isArray(params)) { - seriesParams = params; - } - if (seriesParams) { - appendTooltipItem(result, seriesParams, series); - } else if (Array.isArray(params)) { - for (seriesParams of params) { - appendTooltipItem(result, seriesParams, series); - } - } - return result; -}; - -const appendTooltipItem = (tooltipParams: TooltipParams, seriesParams: CallbackDataParams, series?: EChartsSeriesItem[]) => { - const dataItem = series?.find(s => s.id === seriesParams.seriesId); - const tooltipItem: TooltipItem = { - param: seriesParams, - dataItem - }; - if (dataItem?.comparisonItem) { - tooltipParams.comparisonItems.push(tooltipItem); - } else { - tooltipParams.items.push(tooltipItem); - } -}; - -const constructEchartsTooltipDateElement = (renderer: Renderer2, - tooltipDateFormat: DateFormatProcessor, - settings: EChartsTooltipWidgetSettings, - param: CallbackDataParams, - interval?: Interval): HTMLElement => { - const dateElement: HTMLElement = renderer.createElement('div'); - let dateText: string; - const startTs = param.value[2]; - const endTs = param.value[3]; - if (settings.tooltipDateInterval && startTs && endTs && (endTs - 1) > startTs) { - const startDateText = tooltipDateFormat.update(startTs, interval); - const endDateText = tooltipDateFormat.update(endTs - 1, interval); - if (startDateText === endDateText) { - dateText = startDateText; - } else { - dateText = startDateText + ' - ' + endDateText; - } - } else { - const ts = param.value[0]; - dateText = tooltipDateFormat.update(ts, interval); - } - renderer.appendChild(dateElement, renderer.createText(dateText)); - renderer.setStyle(dateElement, 'font-family', settings.tooltipDateFont.family); - renderer.setStyle(dateElement, 'font-size', settings.tooltipDateFont.size + settings.tooltipDateFont.sizeUnit); - renderer.setStyle(dateElement, 'font-style', settings.tooltipDateFont.style); - renderer.setStyle(dateElement, 'font-weight', settings.tooltipDateFont.weight); - renderer.setStyle(dateElement, 'line-height', settings.tooltipDateFont.lineHeight); - renderer.setStyle(dateElement, 'color', settings.tooltipDateColor); - return dateElement; -}; - -const constructEchartsTooltipSeriesElement = (renderer: Renderer2, - settings: EChartsTooltipWidgetSettings, - item: TooltipItem, - valueFormatFunction: EChartsTooltipValueFormatFunction): HTMLElement => { - const labelValueElement: HTMLElement = renderer.createElement('div'); - renderer.setStyle(labelValueElement, 'display', 'flex'); - renderer.setStyle(labelValueElement, 'flex-direction', 'row'); - renderer.setStyle(labelValueElement, 'align-items', 'center'); - renderer.setStyle(labelValueElement, 'align-self', 'stretch'); - renderer.setStyle(labelValueElement, 'gap', '12px'); - const labelElement: HTMLElement = renderer.createElement('div'); - renderer.setStyle(labelElement, 'display', 'flex'); - renderer.setStyle(labelElement, 'align-items', 'center'); - renderer.setStyle(labelElement, 'gap', '8px'); - renderer.appendChild(labelValueElement, labelElement); - const circleElement: HTMLElement = renderer.createElement('div'); - renderer.setStyle(circleElement, 'width', '8px'); - renderer.setStyle(circleElement, 'height', '8px'); - renderer.setStyle(circleElement, 'border-radius', '50%'); - renderer.setStyle(circleElement, 'background', item.param.color); - renderer.appendChild(labelElement, circleElement); - const labelTextElement: HTMLElement = renderer.createElement('div'); - renderer.appendChild(labelTextElement, renderer.createText(item.param.seriesName)); - renderer.setStyle(labelTextElement, 'font-family', settings.tooltipLabelFont.family); - renderer.setStyle(labelTextElement, 'font-size', settings.tooltipLabelFont.size + settings.tooltipLabelFont.sizeUnit); - renderer.setStyle(labelTextElement, 'font-style', settings.tooltipLabelFont.style); - renderer.setStyle(labelTextElement, 'font-weight', settings.tooltipLabelFont.weight); - renderer.setStyle(labelTextElement, 'line-height', settings.tooltipLabelFont.lineHeight); - renderer.setStyle(labelTextElement, 'color', settings.tooltipLabelColor); - renderer.appendChild(labelElement, labelTextElement); - const valueElement: HTMLElement = renderer.createElement('div'); - let formatFunction = valueFormatFunction; - let latestData: FormattedData; - let units = ''; - let decimals = 0; - if (item.dataItem) { - if (item.dataItem.tooltipValueFormatFunction) { - formatFunction = item.dataItem.tooltipValueFormatFunction; - } - latestData = item.dataItem.latestData; - units = item.dataItem.units; - decimals = item.dataItem.decimals; - } - if (!latestData) { - latestData = {} as FormattedData; - } - const value = formatFunction(item.param.value[1], latestData, units, decimals); - renderer.appendChild(valueElement, renderer.createText(value)); - renderer.setStyle(valueElement, 'flex', '1'); - renderer.setStyle(valueElement, 'text-align', 'end'); - renderer.setStyle(valueElement, 'font-family', settings.tooltipValueFont.family); - renderer.setStyle(valueElement, 'font-size', settings.tooltipValueFont.size + settings.tooltipValueFont.sizeUnit); - renderer.setStyle(valueElement, 'font-style', settings.tooltipValueFont.style); - renderer.setStyle(valueElement, 'font-weight', settings.tooltipValueFont.weight); - renderer.setStyle(valueElement, 'line-height', settings.tooltipValueFont.lineHeight); - renderer.setStyle(valueElement, 'color', settings.tooltipValueColor); - renderer.appendChild(labelValueElement, valueElement); - return labelValueElement; -}; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.models.ts index c99cd5e773..22de664154 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/latest-chart.models.ts @@ -15,11 +15,11 @@ /// import { DataKey, Datasource, LegendPosition } from '@shared/models/widget.models'; -import { BackgroundSettings, Font } from '@shared/models/widget-settings.models'; +import { BackgroundSettings, BackgroundType, Font } from '@shared/models/widget-settings.models'; import { Renderer2 } from '@angular/core'; import { CallbackDataParams } from 'echarts/types/dist/shared'; -import { formatValue, isDefinedAndNotNull } from '@core/utils'; -import { EChartsAnimationSettings } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { formatValue, isDefinedAndNotNull, mergeDeep } from '@core/utils'; +import { chartAnimationDefaultSettings, ChartAnimationSettings } from '@home/components/widget/lib/chart/chart.models'; export interface LatestChartDataItem { id: number; @@ -63,14 +63,40 @@ export interface LatestChartTooltipSettings { tooltipBackgroundBlur: number; } +export const latestChartTooltipDefaultSettings: LatestChartTooltipSettings = { + showTooltip: true, + tooltipValueType: LatestChartTooltipValueType.percentage, + tooltipValueDecimals: 0, + tooltipValueFont: { + family: 'Roboto', + size: 13, + sizeUnit: 'px', + style: 'normal', + weight: '500', + lineHeight: '16px' + }, + tooltipValueColor: 'rgba(0, 0, 0, 0.76)', + tooltipBackgroundColor: 'rgba(255, 255, 255, 0.76)', + tooltipBackgroundBlur: 4 +}; + export interface LatestChartSettings extends LatestChartTooltipSettings { autoScale?: boolean; sortSeries: boolean; showTotal?: boolean; showLegend: boolean; - animation: EChartsAnimationSettings; + animation: ChartAnimationSettings; } +export const latestChartDefaultSettings: LatestChartSettings = { + ...latestChartTooltipDefaultSettings, + autoScale: false, + sortSeries: false, + showTotal: false, + showLegend: true, + animation: mergeDeep({} as ChartAnimationSettings, chartAnimationDefaultSettings) +}; + export interface LatestChartWidgetSettings extends LatestChartSettings { legendPosition: LegendPosition; legendLabelFont: Font; @@ -80,6 +106,39 @@ export interface LatestChartWidgetSettings extends LatestChartSettings { background: BackgroundSettings; } +export const latestChartWidgetDefaultSettings: LatestChartWidgetSettings = { + ...latestChartDefaultSettings, + showLegend: true, + legendPosition: LegendPosition.bottom, + legendLabelFont: { + family: 'Roboto', + size: 12, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '16px' + }, + legendLabelColor: 'rgba(0, 0, 0, 0.38)', + legendValueFont: { + family: 'Roboto', + size: 14, + sizeUnit: 'px', + style: 'normal', + weight: '500', + lineHeight: '20px' + }, + legendValueColor: 'rgba(0, 0, 0, 0.87)', + background: { + type: BackgroundType.color, + color: '#fff', + overlay: { + enabled: false, + color: 'rgba(255,255,255,0.72)', + blur: 3 + } + } +}; + export const latestChartTooltipFormatter = (renderer: Renderer2, settings: LatestChartTooltipSettings, params: CallbackDataParams, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart-widget.models.ts index 05dbef0959..fe6cc158e8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart-widget.models.ts @@ -15,19 +15,14 @@ /// import { - LatestChartTooltipValueType, + latestChartWidgetDefaultSettings, LatestChartWidgetSettings } from '@home/components/widget/lib/chart/latest-chart.models'; -import { BackgroundType, Font } from '@shared/models/widget-settings.models'; -import { LegendPosition } from '@shared/models/widget.models'; +import { Font } from '@shared/models/widget-settings.models'; import { DeepPartial } from '@shared/models/common'; -import { - pieChartAnimationDefaultSettings, - PieChartLabelPosition, - PieChartSettings -} from '@home/components/widget/lib/chart/pie-chart.models'; +import { pieChartAnimationDefaultSettings, PieChartSettings } from '@home/components/widget/lib/chart/pie-chart.models'; import { isDefinedAndNotNull, mergeDeep } from '@core/utils'; -import { EChartsAnimationSettings } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { ChartAnimationSettings, PieChartLabelPosition } from '@home/components/widget/lib/chart/chart.models'; export interface PieChartWidgetSettings extends LatestChartWidgetSettings { showLabel: boolean; @@ -41,6 +36,9 @@ export interface PieChartWidgetSettings extends LatestChartWidgetSettings { } export const pieChartWidgetDefaultSettings: PieChartWidgetSettings = { + ...latestChartWidgetDefaultSettings, + animation: mergeDeep({} as ChartAnimationSettings, + pieChartAnimationDefaultSettings), showLabel: true, labelPosition: PieChartLabelPosition.outside, labelFont: { @@ -55,53 +53,7 @@ export const pieChartWidgetDefaultSettings: PieChartWidgetSettings = { borderWidth: 0, borderColor: '#000', radius: 80, - clockwise: false, - sortSeries: false, - animation: mergeDeep({} as EChartsAnimationSettings, - pieChartAnimationDefaultSettings), - showLegend: true, - legendPosition: LegendPosition.bottom, - legendLabelFont: { - family: 'Roboto', - size: 12, - sizeUnit: 'px', - style: 'normal', - weight: '400', - lineHeight: '16px' - }, - legendLabelColor: 'rgba(0, 0, 0, 0.38)', - legendValueFont: { - family: 'Roboto', - size: 14, - sizeUnit: 'px', - style: 'normal', - weight: '500', - lineHeight: '20px' - }, - legendValueColor: 'rgba(0, 0, 0, 0.87)', - showTooltip: true, - tooltipValueType: LatestChartTooltipValueType.percentage, - tooltipValueDecimals: 0, - tooltipValueFont: { - family: 'Roboto', - size: 13, - sizeUnit: 'px', - style: 'normal', - weight: '500', - lineHeight: '16px' - }, - tooltipValueColor: 'rgba(0, 0, 0, 0.76)', - tooltipBackgroundColor: 'rgba(255, 255, 255, 0.76)', - tooltipBackgroundBlur: 4, - background: { - type: BackgroundType.color, - color: '#fff', - overlay: { - enabled: false, - color: 'rgba(255,255,255,0.72)', - blur: 3 - } - } + clockwise: false }; export const pieChartWidgetPieChartSettings = (settings: PieChartWidgetSettings): DeepPartial => ({ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.models.ts index 419c4b61bb..ebe752eb4b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/pie-chart.models.ts @@ -15,29 +15,13 @@ /// import { ColorSettings, constantColor, Font } from '@shared/models/widget-settings.models'; -import { - LatestChartSettings, - LatestChartTooltipValueType -} from '@home/components/widget/lib/chart/latest-chart.models'; +import { latestChartDefaultSettings, LatestChartSettings } from '@home/components/widget/lib/chart/latest-chart.models'; import { mergeDeep } from '@core/utils'; import { - EChartsAnimationEasing, - EChartsAnimationSettings -} from '@home/components/widget/lib/chart/echarts-widget.models'; - -export enum PieChartLabelPosition { - outside = 'outside', - inside = 'inside' -} - -export const pieChartLabelPositions = Object.keys(PieChartLabelPosition) as PieChartLabelPosition[]; - -export const pieChartLabelPositionTranslations = new Map( - [ - [PieChartLabelPosition.outside, 'widgets.pie-chart.label-position-outside'], - [PieChartLabelPosition.inside, 'widgets.pie-chart.label-position-inside'] - ] -); + chartAnimationDefaultSettings, + ChartAnimationSettings, + PieChartLabelPosition +} from '@home/components/widget/lib/chart/chart.models'; export interface PieChartSettings extends LatestChartSettings { doughnut: boolean; @@ -59,27 +43,19 @@ export interface PieChartSettings extends LatestChartSettings { emphasisShadowColor: string; } -export const pieChartAnimationDefaultSettings: EChartsAnimationSettings = { - animation: true, - animationThreshold: 2000, - animationDuration: 1000, - animationEasing: EChartsAnimationEasing.cubicOut, - animationDelay: 0, - animationDurationUpdate: 500, - animationEasingUpdate: EChartsAnimationEasing.cubicOut, - animationDelayUpdate: 0 -}; +export const pieChartAnimationDefaultSettings: ChartAnimationSettings = + mergeDeep({} as ChartAnimationSettings, chartAnimationDefaultSettings, { + animationDuration: 1000, + animationDurationUpdate: 500 + } as ChartAnimationSettings); export const pieChartDefaultSettings: PieChartSettings = { - autoScale: false, + ...latestChartDefaultSettings, + animation: mergeDeep({} as ChartAnimationSettings, + pieChartAnimationDefaultSettings), doughnut: false, radius: '80%', clockwise: false, - sortSeries: false, - showTotal: false, - animation: mergeDeep({} as EChartsAnimationSettings, - pieChartAnimationDefaultSettings), - showLegend: true, totalValueFont: { family: 'Roboto', size: 24, @@ -107,20 +83,5 @@ export const pieChartDefaultSettings: PieChartSettings = { emphasisBorderWidth: 0, emphasisBorderColor: '#000', emphasisShadowBlur: 10, - emphasisShadowColor: 'rgba(0, 0, 0, 0.5)', - showTooltip: true, - tooltipValueType: LatestChartTooltipValueType.percentage, - tooltipValueDecimals: 0, - tooltipValueFont: { - family: 'Roboto', - size: 13, - sizeUnit: 'px', - style: 'normal', - weight: '500', - lineHeight: '16px' - }, - tooltipValueColor: 'rgba(0, 0, 0, 0.76)', - tooltipBackgroundColor: 'rgba(255, 255, 255, 0.76)', - tooltipBackgroundBlur: 4 + emphasisShadowColor: 'rgba(0, 0, 0, 0.5)' }; - diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.component.ts new file mode 100644 index 0000000000..07cffaefab --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.component.ts @@ -0,0 +1,75 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, TemplateRef, ViewChild, ViewEncapsulation } from '@angular/core'; +import { WidgetContext } from '@home/models/widget-component.models'; +import { WidgetComponent } from '@home/components/widget/widget.component'; +import { TranslateService } from '@ngx-translate/core'; +import { + LatestChartComponent, + LatestChartComponentCallbacks +} from '@home/components/widget/lib/chart/latest-chart.component'; +import { TbBarsChart } from '@home/components/widget/lib/chart/bars-chart'; +import { + polarAreaChartWidgetBarsChartSettings, + polarAreaChartWidgetDefaultSettings, + PolarAreaChartWidgetSettings +} from '@home/components/widget/lib/chart/polar-area-widget.models'; + +@Component({ + selector: 'tb-polar-area-chart-widget', + templateUrl: './latest-chart-widget.component.html', + styleUrls: [], + encapsulation: ViewEncapsulation.None +}) +export class PolarAreaWidgetComponent implements OnInit { + + @ViewChild('latestChart') + latestChart: LatestChartComponent; + + @Input() + ctx: WidgetContext; + + @Input() + widgetTitlePanel: TemplateRef; + + settings: PolarAreaChartWidgetSettings; + + callbacks: LatestChartComponentCallbacks; + + constructor(private widgetComponent: WidgetComponent, + private translate: TranslateService) { + } + + ngOnInit(): void { + this.ctx.$scope.polarAreaChartWidget = this; + this.settings = {...polarAreaChartWidgetDefaultSettings, ...this.ctx.settings}; + this.callbacks = { + createChart: (chartShape, renderer) => { + const settings = polarAreaChartWidgetBarsChartSettings(this.settings); + return new TbBarsChart(this.ctx, settings, chartShape.nativeElement, renderer, this.translate, true); + } + }; + } + + public onInit() { + this.latestChart?.onInit(); + } + + public onDataUpdated() { + this.latestChart?.onDataUpdated(); + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.models.ts new file mode 100644 index 0000000000..6ef08f631e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/polar-area-widget.models.ts @@ -0,0 +1,82 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + latestChartWidgetDefaultSettings, + LatestChartWidgetSettings +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { Font } from '@shared/models/widget-settings.models'; +import { + ChartAnimationSettings, + chartBarDefaultSettings, + ChartBarSettings, + chartColorScheme, + PieChartLabelPosition +} from '@home/components/widget/lib/chart/chart.models'; +import { mergeDeep } from '@core/utils'; +import { + barsChartAnimationDefaultSettings, + BarsChartSettings +} from '@home/components/widget/lib/chart/bars-chart.models'; +import { DeepPartial } from '@shared/models/common'; + +export interface PolarAreaChartWidgetSettings extends LatestChartWidgetSettings { + axisMin?: number; + axisMax?: number; + axisTickLabelFont: Font; + axisTickLabelColor: string; + angleAxisStartAngle?: number; + barSettings: ChartBarSettings; +} + +export const polarAreaChartWidgetDefaultSettings: PolarAreaChartWidgetSettings = { + ...latestChartWidgetDefaultSettings, + animation: mergeDeep({} as ChartAnimationSettings, + barsChartAnimationDefaultSettings), + axisTickLabelFont: { + family: 'Roboto', + size: 12, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '1' + }, + axisTickLabelColor: chartColorScheme['axis.tickLabel'].light, + angleAxisStartAngle: 90, + barSettings: mergeDeep({} as ChartBarSettings, chartBarDefaultSettings, + {barWidth: 100, showLabel: true, labelPosition: PieChartLabelPosition.outside} as ChartBarSettings) +}; + +export const polarAreaChartWidgetBarsChartSettings = (settings: PolarAreaChartWidgetSettings): DeepPartial => ({ + polar: true, + axisMin: settings.axisMin, + axisMax: settings.axisMax, + axisTickLabelFont: settings.axisTickLabelFont, + axisTickLabelColor: settings.axisTickLabelColor, + angleAxisStartAngle: settings.angleAxisStartAngle, + barSettings: settings.barSettings, + sortSeries: settings.sortSeries, + showTotal: false, + animation: settings.animation, + showLegend: settings.showLegend, + showTooltip: settings.showTooltip, + tooltipValueType: settings.tooltipValueType, + tooltipValueDecimals: settings.tooltipValueDecimals, + tooltipValueFont: settings.tooltipValueFont, + tooltipValueColor: settings.tooltipValueColor, + tooltipBackgroundColor: settings.tooltipBackgroundColor, + tooltipBackgroundBlur: settings.tooltipBackgroundBlur +}); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.models.ts index c411737761..45252bd7f6 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/range-chart-widget.models.ts @@ -24,32 +24,36 @@ import { sortedColorRange } from '@shared/models/widget-settings.models'; import { LegendPosition } from '@shared/models/widget.models'; -import { - echartsAnimationDefaultSettings, - EChartsAnimationSettings, - EChartsShape, - EChartsTooltipWidgetSettings -} from '@home/components/widget/lib/chart/echarts-widget.models'; import { createTimeSeriesChartVisualMapPiece, defaultTimeSeriesChartXAxisSettings, defaultTimeSeriesChartYAxisSettings, LineSeriesStepType, - SeriesFillType, - SeriesLabelPosition, ThresholdLabelPosition, - timeSeriesChartColorScheme, timeSeriesChartGridDefaultSettings, TimeSeriesChartGridSettings, + ThresholdLabelPosition, + timeSeriesChartGridDefaultSettings, + TimeSeriesChartGridSettings, TimeSeriesChartKeySettings, TimeSeriesChartLineType, TimeSeriesChartSeriesType, TimeSeriesChartSettings, - TimeSeriesChartThreshold, timeSeriesChartThresholdDefaultSettings, + TimeSeriesChartThreshold, + timeSeriesChartThresholdDefaultSettings, TimeSeriesChartThresholdType, + TimeSeriesChartTooltipWidgetSettings, TimeSeriesChartVisualMapPiece, TimeSeriesChartXAxisSettings, TimeSeriesChartYAxisSettings } from '@home/components/widget/lib/chart/time-series-chart.models'; import { isNumber, mergeDeep } from '@core/utils'; import { DeepPartial } from '@shared/models/common'; +import { + chartAnimationDefaultSettings, + ChartAnimationSettings, + chartColorScheme, + ChartFillType, + ChartLabelPosition, + ChartShape +} from '@home/components/widget/lib/chart/chart.models'; export interface RangeItem { index: number; @@ -62,7 +66,7 @@ export interface RangeItem { piece: TimeSeriesChartVisualMapPiece; } -export interface RangeChartWidgetSettings extends EChartsTooltipWidgetSettings { +export interface RangeChartWidgetSettings extends TimeSeriesChartTooltipWidgetSettings { dataZoom: boolean; rangeColors: Array; outOfRangeColor: string; @@ -78,17 +82,17 @@ export interface RangeChartWidgetSettings extends EChartsTooltipWidgetSettings { lineWidth: number; showPoints: boolean; showPointLabel: boolean; - pointLabelPosition: SeriesLabelPosition; + pointLabelPosition: ChartLabelPosition; pointLabelFont: Font; pointLabelColor: string; enablePointLabelBackground: boolean; pointLabelBackground: string; - pointShape: EChartsShape; + pointShape: ChartShape; pointSize: number; grid: TimeSeriesChartGridSettings; yAxis: TimeSeriesChartYAxisSettings; xAxis: TimeSeriesChartXAxisSettings; - animation: EChartsAnimationSettings; + animation: ChartAnimationSettings; thresholds: TimeSeriesChartThreshold[]; showLegend: boolean; legendPosition: LegendPosition; @@ -115,9 +119,9 @@ export const rangeChartDefaultSettings: RangeChartWidgetSettings = { timeSeriesChartThresholdDefaultSettings, { lineColor: '#37383b', lineType: TimeSeriesChartLineType.dashed, - startSymbol: EChartsShape.circle, + startSymbol: ChartShape.circle, startSymbolSize: 5, - endSymbol: EChartsShape.arrow, + endSymbol: ChartShape.arrow, endSymbolSize: 7, labelPosition: ThresholdLabelPosition.insideEndTop, labelColor: '#37383b', @@ -132,7 +136,7 @@ export const rangeChartDefaultSettings: RangeChartWidgetSettings = { lineWidth: 2, showPoints: false, showPointLabel: false, - pointLabelPosition: SeriesLabelPosition.top, + pointLabelPosition: ChartLabelPosition.top, pointLabelFont: { family: 'Roboto', size: 11, @@ -141,10 +145,10 @@ export const rangeChartDefaultSettings: RangeChartWidgetSettings = { weight: '400', lineHeight: '1' }, - pointLabelColor: timeSeriesChartColorScheme['series.label'].light, + pointLabelColor: chartColorScheme['series.label'].light, enablePointLabelBackground: false, pointLabelBackground: 'rgba(255,255,255,0.56)', - pointShape: EChartsShape.emptyCircle, + pointShape: ChartShape.emptyCircle, pointSize: 4, grid: mergeDeep({} as TimeSeriesChartGridSettings, timeSeriesChartGridDefaultSettings), @@ -154,8 +158,8 @@ export const rangeChartDefaultSettings: RangeChartWidgetSettings = { xAxis: mergeDeep({} as TimeSeriesChartXAxisSettings, defaultTimeSeriesChartXAxisSettings, {showSplitLines: false} as TimeSeriesChartXAxisSettings), - animation: mergeDeep({} as EChartsAnimationSettings, - echartsAnimationDefaultSettings), + animation: mergeDeep({} as ChartAnimationSettings, + chartAnimationDefaultSettings), thresholds: [], showLegend: true, legendPosition: LegendPosition.top, @@ -279,7 +283,7 @@ export const rangeChartTimeSeriesKeySettings = (settings: RangeChartWidgetSettin pointShape: settings.pointShape, pointSize: settings.pointSize, fillAreaSettings: { - type: settings.fillArea ? SeriesFillType.opacity : SeriesFillType.none, + type: settings.fillArea ? ChartFillType.opacity : ChartFillType.none, opacity: settings.fillAreaOpacity } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-bar.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-bar.models.ts index f54a803d12..22d2a26473 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-bar.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-bar.models.ts @@ -16,7 +16,7 @@ import { LinearGradientObject } from 'zrender/lib/graphic/LinearGradient'; import { Interval, IntervalMath } from '@shared/models/time/time.models'; -import { LabelFormatterCallback, SeriesLabelOption } from 'echarts/types/src/util/types'; +import { LabelFormatterCallback } from 'echarts/types/src/util/types'; import { TimeSeriesChartDataItem, TimeSeriesChartNoAggregationBarWidthStrategy @@ -25,6 +25,7 @@ import { CustomSeriesRenderItemParams } from 'echarts'; import { CallbackDataParams, CustomSeriesRenderItemAPI, CustomSeriesRenderItemReturn } from 'echarts/types/dist/shared'; import { isNumeric } from '@core/utils'; import * as echarts from 'echarts/core'; +import { BarSeriesLabelOption } from 'echarts/types/src/chart/bar/BarSeries'; export interface BarVisualSettings { color: string | LinearGradientObject; @@ -48,7 +49,7 @@ export interface BarRenderContext { barIndex?: number; noAggregation?: boolean; visualSettings?: BarVisualSettings; - labelOption?: SeriesLabelOption; + labelOption?: BarSeriesLabelOption; additionalLabelOption?: {[key: string]: any}; barStackIndex?: number; currentStackItems?: TimeSeriesChartDataItem[]; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-state.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-state.models.ts index 9a72361632..35159ba944 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-state.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-state.models.ts @@ -18,10 +18,10 @@ import { TimeSeriesChartStateSettings, TimeSeriesChartStateSourceType, TimeSeriesChartTicksFormatter, - TimeSeriesChartTicksGenerator + TimeSeriesChartTicksGenerator, + TimeSeriesChartTooltipValueFormatFunction } from '@home/components/widget/lib/chart/time-series-chart.models'; import { UtilsService } from '@core/services/utils.service'; -import { EChartsTooltipValueFormatFunction } from '@home/components/widget/lib/chart/echarts-widget.models'; import { FormattedData } from '@shared/models/widget.models'; import { formatValue, isDefinedAndNotNull, isNumber, isNumeric } from '@core/utils'; import { LabelFormatterCallback } from 'echarts'; @@ -35,7 +35,7 @@ export class TimeSeriesChartStateValueConverter { public readonly ticksGenerator: TimeSeriesChartTicksGenerator; public readonly ticksFormatter: TimeSeriesChartTicksFormatter; - public readonly tooltipFormatter: EChartsTooltipValueFormatFunction; + public readonly tooltipFormatter: TimeSeriesChartTooltipValueFormatFunction; public readonly labelFormatter: LabelFormatterCallback; public readonly valueConverter: (value: any) => any; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts index 81ea480531..48a328613e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.component.ts @@ -27,7 +27,10 @@ import { ViewChild, ViewEncapsulation } from '@angular/core'; -import { timeSeriesChartKeyDefaultSettings, TimeSeriesChartKeySettings } from '@home/components/widget/lib/chart/time-series-chart.models'; +import { + timeSeriesChartKeyDefaultSettings, + TimeSeriesChartKeySettings +} from '@home/components/widget/lib/chart/time-series-chart.models'; import { WidgetContext } from '@home/models/widget-component.models'; import { Observable } from 'rxjs'; import { backgroundStyle, ComponentStyle, overlayStyle, textStyle } from '@shared/models/widget-settings.models'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.models.ts index 00abd27c33..6ee3409269 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart-widget.models.ts @@ -14,7 +14,10 @@ /// limitations under the License. /// -import { timeSeriesChartDefaultSettings, TimeSeriesChartSettings } from '@home/components/widget/lib/chart/time-series-chart.models'; +import { + timeSeriesChartDefaultSettings, + TimeSeriesChartSettings +} from '@home/components/widget/lib/chart/time-series-chart.models'; import { BackgroundSettings, BackgroundType, Font } from '@shared/models/widget-settings.models'; import { defaultLegendConfig, LegendConfig, LegendPosition, widgetType } from '@shared/models/widget.models'; import { mergeDeep } from '@core/utils'; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts index 0fd702a95f..a906fe0bed 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.models.ts @@ -16,27 +16,21 @@ import { ECharts, - echartsAnimationDefaultSettings, - EChartsAnimationSettings, EChartsOption, - EChartsSeriesItem, - EChartsShape, - EChartsTooltipTrigger, - EChartsTooltipValueFormatFunction, - EChartsTooltipWidgetSettings, - measureThresholdOffset, - timeAxisBandWidthCalculator + measureThresholdOffset } from '@home/components/widget/lib/chart/echarts-widget.models'; import { autoDateFormat, AutoDateFormatSettings, ComponentStyle, + DateFormatProcessor, + DateFormatSettings, Font, - textStyle, tsToFormatTimeUnit } from '@shared/models/widget-settings.models'; import { - LabelLayoutOptionCallback, + CallbackDataParams, + TimeAxisBandWidthCalculator, VisualMapComponentOption, XAXisOption, YAXisOption @@ -53,10 +47,9 @@ import { mergeDeep, parseFunction } from '@core/utils'; -import { LinearGradientObject } from 'zrender/lib/graphic/LinearGradient'; import tinycolor from 'tinycolor2'; -import { ValueAxisBaseOption } from 'echarts/types/src/coord/axisCommonTypes'; -import { LabelLayoutOption, SeriesLabelOption } from 'echarts/types/src/util/types'; +import { TimeAxisBaseOption, ValueAxisBaseOption } from 'echarts/types/src/coord/axisCommonTypes'; +import { SeriesLabelOption } from 'echarts/types/src/util/types'; import { LabelFormatterCallback } from 'echarts'; import { BarRenderContext, @@ -64,16 +57,402 @@ import { BarVisualSettings, renderTimeSeriesBar } from '@home/components/widget/lib/chart/time-series-chart-bar.models'; -import { DataKey, DataKeySettingsWithComparison, WidgetComparisonSettings } from '@shared/models/widget.models'; +import { + DataEntry, + DataKey, + DataKeySettingsWithComparison, + DataSet, + Datasource, + FormattedData, + WidgetComparisonSettings +} from '@shared/models/widget.models'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; -import { TbColorScheme } from '@shared/models/color.models'; import { AbstractControl, ValidationErrors } from '@angular/forms'; import { MarkLine2DDataItemOption } from 'echarts/types/src/component/marker/MarkLineModel'; import { DatePipe } from '@angular/common'; import { BuiltinTextPosition } from 'zrender/src/core/types'; import { CartesianAxisOption } from 'echarts/types/src/coord/cartesian/AxisModel'; -import { WidgetTimewindow } from '@shared/models/time/time.models'; +import { + calculateAggIntervalWithWidgetTimeWindow, + Interval, + IntervalMath, + WidgetTimewindow +} from '@shared/models/time/time.models'; import { UtilsService } from '@core/services/utils.service'; +import { Renderer2 } from '@angular/core'; +import { + chartAnimationDefaultSettings, + ChartAnimationSettings, + chartBarDefaultSettings, + ChartBarSettings, + chartColorScheme, + ChartFillSettings, + ChartFillType, + ChartLabelPosition, + ChartShape, + createChartTextStyle, + createLinearOpacityGradient, + PieChartLabelPosition, + prepareChartThemeColor +} from '@home/components/widget/lib/chart/chart.models'; +import { BarSeriesLabelOption } from 'echarts/types/src/chart/bar/BarSeries'; + +type TimeSeriesChartDataEntry = [number, any, number, number]; + +type TimeSeriesChartDataSet = {name: string; value: TimeSeriesChartDataEntry}[]; + +export const toTimeSeriesChartDataSet = (data: DataSet, valueConverter?: (value: any) => any): TimeSeriesChartDataSet => { + if (!data?.length) { + return []; + } else { + return data.map(d => { + const ts = isDefinedAndNotNull(d[2]) ? d[2][0] : d[0]; + return { + name: ts + '', + value: toTimeSeriesChartDataEntry(d, valueConverter) + }; + }); + } +}; + +const toTimeSeriesChartDataEntry = (entry: DataEntry, valueConverter?: (value: any) => any): TimeSeriesChartDataEntry => { + const value = valueConverter ? valueConverter(entry[1]) : entry[1]; + const item: TimeSeriesChartDataEntry = [entry[0], value, entry[0], entry[0]]; + if (isDefinedAndNotNull(entry[2])) { + item[2] = entry[2][0]; + item[3] = entry[2][1]; + } + return item; +}; + +export type TimeSeriesChartTooltipValueFormatFunction = + (value: any, latestData: FormattedData, units?: string, decimals?: number) => string; + +export interface TimeSeriesChartDataItem { + id: string; + datasource: Datasource; + dataKey: DataKey; + data: TimeSeriesChartDataSet; + dataSet?: DataSet; + enabled: boolean; + units?: string; + decimals?: number; + latestData?: FormattedData; + tooltipValueFormatFunction?: TimeSeriesChartTooltipValueFormatFunction; + comparisonItem?: boolean; + xAxisIndex: number; + yAxisId: TimeSeriesChartYAxisId; + yAxisIndex: number; + option?: LineSeriesOption | CustomSeriesOption; + barRenderContext?: BarRenderContext; +} + +export const timeAxisBandWidthCalculator: TimeAxisBandWidthCalculator = (model) => { + let interval: number; + const axisOption = model.option; + const seriesDataIndices = axisOption.axisPointer?.seriesDataIndices; + if (seriesDataIndices?.length) { + const seriesDataIndex = seriesDataIndices[0]; + const series = model.ecModel.getSeriesByIndex(seriesDataIndex.seriesIndex); + if (series) { + const values = series.getData().getValues(seriesDataIndex.dataIndex); + const start = values[2]; + const end = values[3]; + if (typeof start === 'number' && typeof end === 'number') { + interval = Math.max(end - start, 1); + } + } + } + if (!interval) { + const tbTimeWindow: WidgetTimewindow = (axisOption as any).tbTimeWindow; + if (isDefinedAndNotNull(tbTimeWindow)) { + if (axisOption.axisPointer?.value && typeof axisOption.axisPointer?.value === 'number') { + const intervalArray = calculateAggIntervalWithWidgetTimeWindow(tbTimeWindow, axisOption.axisPointer.value); + const start = intervalArray[0]; + const end = intervalArray[1]; + interval = Math.max(end - start, 1); + } else { + interval = IntervalMath.numberValue(tbTimeWindow.interval); + } + } + } + if (interval) { + const timeScale = model.axis.scale; + const axisExtent = model.axis.getExtent(); + const dataExtent = timeScale.getExtent(); + const size = Math.abs(axisExtent[1] - axisExtent[0]); + return interval * (size / (dataExtent[1] - dataExtent[0])); + } +}; + + +const minDataTs = (dataSet: TimeSeriesChartDataSet): number => dataSet.length ? dataSet.map(data => + Number(data.name)).reduce((a, b) => Math.min(a, b)) : undefined; + +const maxDataTs = (dataSet: TimeSeriesChartDataSet): number => dataSet.length ? dataSet.map(data => + Number(data.name)).reduce((a, b) => Math.max(a, b)) : undefined; + +export const adjustTimeAxisExtentToData = (timeAxisOption: TimeAxisBaseOption, + dataItems: TimeSeriesChartDataItem[], + defaultMin: number, + defaultMax: number): void => { + let min: number; + let max: number; + for (const item of dataItems) { + if (item.enabled) { + const minTs = minDataTs(item.data); + if (typeof minTs !== 'undefined') { + min = (typeof min !== 'undefined') ? Math.min(min, minTs) : minTs; + } + const maxTs = maxDataTs(item.data); + if (typeof maxTs !== 'undefined') { + max = (typeof max !== 'undefined') ? Math.max(max, maxTs) : maxTs; + } + } + } + timeAxisOption.min = (typeof min !== 'undefined' && Math.abs(min - defaultMin) < 1000) ? min : defaultMin; + timeAxisOption.max = (typeof max !== 'undefined' && Math.abs(max - defaultMax) < 1000) ? max : defaultMax; +}; + +export enum TimeSeriesChartTooltipTrigger { + point = 'point', + axis = 'axis' +} + +export const tooltipTriggerTranslationMap = new Map( + [ + [ TimeSeriesChartTooltipTrigger.point, 'tooltip.trigger-point' ], + [ TimeSeriesChartTooltipTrigger.axis, 'tooltip.trigger-axis' ] + ] +); + +export interface TimeSeriesChartTooltipWidgetSettings { + showTooltip: boolean; + tooltipTrigger?: TimeSeriesChartTooltipTrigger; + tooltipShowFocusedSeries?: boolean; + tooltipLabelFont: Font; + tooltipLabelColor: string; + tooltipValueFont: Font; + tooltipValueColor: string; + tooltipValueFormatter?: string | TimeSeriesChartTooltipValueFormatFunction; + tooltipShowDate: boolean; + tooltipDateInterval?: boolean; + tooltipDateFormat: DateFormatSettings; + tooltipDateFont: Font; + tooltipDateColor: string; + tooltipBackgroundColor: string; + tooltipBackgroundBlur: number; +} + +export const createTooltipValueFormatFunction = + (tooltipValueFormatter: string | TimeSeriesChartTooltipValueFormatFunction): TimeSeriesChartTooltipValueFormatFunction => { + let tooltipValueFormatFunction: TimeSeriesChartTooltipValueFormatFunction; + if (isFunction(tooltipValueFormatter)) { + tooltipValueFormatFunction = tooltipValueFormatter as TimeSeriesChartTooltipValueFormatFunction; + } else if (typeof tooltipValueFormatter === 'string' && tooltipValueFormatter.length) { + try { + tooltipValueFormatFunction = + new Function('value', 'latestData', tooltipValueFormatter) as TimeSeriesChartTooltipValueFormatFunction; + } catch (e) {} + } + return tooltipValueFormatFunction; + }; + +export const timeSeriesChartTooltipFormatter = (renderer: Renderer2, + tooltipDateFormat: DateFormatProcessor, + settings: TimeSeriesChartTooltipWidgetSettings, + params: CallbackDataParams[] | CallbackDataParams, + valueFormatFunction: TimeSeriesChartTooltipValueFormatFunction, + focusedSeriesIndex: number, + series?: TimeSeriesChartDataItem[], + interval?: Interval): null | HTMLElement => { + + const tooltipParams = mapTooltipParams(params, series, focusedSeriesIndex); + if (!tooltipParams.items.length && !tooltipParams.comparisonItems.length) { + return null; + } + + const tooltipElement: HTMLElement = renderer.createElement('div'); + renderer.setStyle(tooltipElement, 'display', 'flex'); + renderer.setStyle(tooltipElement, 'flex-direction', 'column'); + renderer.setStyle(tooltipElement, 'align-items', 'flex-start'); + renderer.setStyle(tooltipElement, 'gap', '16px'); + + buildItemsTooltip(tooltipElement, tooltipParams.items, renderer, tooltipDateFormat, settings, valueFormatFunction, interval); + buildItemsTooltip(tooltipElement, tooltipParams.comparisonItems, renderer, tooltipDateFormat, settings, valueFormatFunction, interval); + + return tooltipElement; +}; + +interface TooltipItem { + param: CallbackDataParams; + dataItem: TimeSeriesChartDataItem; +} + +interface TooltipParams { + items: TooltipItem[]; + comparisonItems: TooltipItem[]; +} + +const buildItemsTooltip = (tooltipElement: HTMLElement, + items: TooltipItem[], + renderer: Renderer2, + tooltipDateFormat: DateFormatProcessor, + settings: TimeSeriesChartTooltipWidgetSettings, + valueFormatFunction: TimeSeriesChartTooltipValueFormatFunction, + interval?: Interval) => { + if (items.length) { + const tooltipItemsElement: HTMLElement = renderer.createElement('div'); + renderer.setStyle(tooltipItemsElement, 'display', 'flex'); + renderer.setStyle(tooltipItemsElement, 'flex-direction', 'column'); + renderer.setStyle(tooltipItemsElement, 'align-items', 'flex-start'); + renderer.setStyle(tooltipItemsElement, 'gap', '4px'); + renderer.appendChild(tooltipElement, tooltipItemsElement); + if (settings.tooltipShowDate) { + renderer.appendChild(tooltipItemsElement, + constructTooltipDateElement(renderer, tooltipDateFormat, settings, items[0].param, interval)); + } + for (const item of items) { + renderer.appendChild(tooltipItemsElement, + constructTooltipSeriesElement(renderer, settings, item, valueFormatFunction)); + } + } +}; + +const mapTooltipParams = (params: CallbackDataParams[] | CallbackDataParams, + series?: TimeSeriesChartDataItem[], + focusedSeriesIndex?: number): TooltipParams => { + const result: TooltipParams = { + items: [], + comparisonItems: [] + }; + if (!params || Array.isArray(params) && !params[0]) { + return result; + } + const firstParam = Array.isArray(params) ? params[0] : params; + if (!firstParam.value) { + return result; + } + let seriesParams: CallbackDataParams = null; + if (Array.isArray(params) && focusedSeriesIndex > -1) { + seriesParams = params.find(param => param.seriesIndex === focusedSeriesIndex); + } else if (!Array.isArray(params)) { + seriesParams = params; + } + if (seriesParams) { + appendTooltipItem(result, seriesParams, series); + } else if (Array.isArray(params)) { + for (seriesParams of params) { + appendTooltipItem(result, seriesParams, series); + } + } + return result; +}; + +const appendTooltipItem = (tooltipParams: TooltipParams, seriesParams: CallbackDataParams, series?: TimeSeriesChartDataItem[]) => { + const dataItem = series?.find(s => s.id === seriesParams.seriesId); + const tooltipItem: TooltipItem = { + param: seriesParams, + dataItem + }; + if (dataItem?.comparisonItem) { + tooltipParams.comparisonItems.push(tooltipItem); + } else { + tooltipParams.items.push(tooltipItem); + } +}; + +const constructTooltipDateElement = (renderer: Renderer2, + tooltipDateFormat: DateFormatProcessor, + settings: TimeSeriesChartTooltipWidgetSettings, + param: CallbackDataParams, + interval?: Interval): HTMLElement => { + const dateElement: HTMLElement = renderer.createElement('div'); + let dateText: string; + const startTs = param.value[2]; + const endTs = param.value[3]; + if (settings.tooltipDateInterval && startTs && endTs && (endTs - 1) > startTs) { + const startDateText = tooltipDateFormat.update(startTs, interval); + const endDateText = tooltipDateFormat.update(endTs - 1, interval); + if (startDateText === endDateText) { + dateText = startDateText; + } else { + dateText = startDateText + ' - ' + endDateText; + } + } else { + const ts = param.value[0]; + dateText = tooltipDateFormat.update(ts, interval); + } + renderer.appendChild(dateElement, renderer.createText(dateText)); + renderer.setStyle(dateElement, 'font-family', settings.tooltipDateFont.family); + renderer.setStyle(dateElement, 'font-size', settings.tooltipDateFont.size + settings.tooltipDateFont.sizeUnit); + renderer.setStyle(dateElement, 'font-style', settings.tooltipDateFont.style); + renderer.setStyle(dateElement, 'font-weight', settings.tooltipDateFont.weight); + renderer.setStyle(dateElement, 'line-height', settings.tooltipDateFont.lineHeight); + renderer.setStyle(dateElement, 'color', settings.tooltipDateColor); + return dateElement; +}; + +const constructTooltipSeriesElement = (renderer: Renderer2, + settings: TimeSeriesChartTooltipWidgetSettings, + item: TooltipItem, + valueFormatFunction: TimeSeriesChartTooltipValueFormatFunction): HTMLElement => { + const labelValueElement: HTMLElement = renderer.createElement('div'); + renderer.setStyle(labelValueElement, 'display', 'flex'); + renderer.setStyle(labelValueElement, 'flex-direction', 'row'); + renderer.setStyle(labelValueElement, 'align-items', 'center'); + renderer.setStyle(labelValueElement, 'align-self', 'stretch'); + renderer.setStyle(labelValueElement, 'gap', '12px'); + const labelElement: HTMLElement = renderer.createElement('div'); + renderer.setStyle(labelElement, 'display', 'flex'); + renderer.setStyle(labelElement, 'align-items', 'center'); + renderer.setStyle(labelElement, 'gap', '8px'); + renderer.appendChild(labelValueElement, labelElement); + const circleElement: HTMLElement = renderer.createElement('div'); + renderer.setStyle(circleElement, 'width', '8px'); + renderer.setStyle(circleElement, 'height', '8px'); + renderer.setStyle(circleElement, 'border-radius', '50%'); + renderer.setStyle(circleElement, 'background', item.param.color); + renderer.appendChild(labelElement, circleElement); + const labelTextElement: HTMLElement = renderer.createElement('div'); + renderer.appendChild(labelTextElement, renderer.createText(item.param.seriesName)); + renderer.setStyle(labelTextElement, 'font-family', settings.tooltipLabelFont.family); + renderer.setStyle(labelTextElement, 'font-size', settings.tooltipLabelFont.size + settings.tooltipLabelFont.sizeUnit); + renderer.setStyle(labelTextElement, 'font-style', settings.tooltipLabelFont.style); + renderer.setStyle(labelTextElement, 'font-weight', settings.tooltipLabelFont.weight); + renderer.setStyle(labelTextElement, 'line-height', settings.tooltipLabelFont.lineHeight); + renderer.setStyle(labelTextElement, 'color', settings.tooltipLabelColor); + renderer.appendChild(labelElement, labelTextElement); + const valueElement: HTMLElement = renderer.createElement('div'); + let formatFunction = valueFormatFunction; + let latestData: FormattedData; + let units = ''; + let decimals = 0; + if (item.dataItem) { + if (item.dataItem.tooltipValueFormatFunction) { + formatFunction = item.dataItem.tooltipValueFormatFunction; + } + latestData = item.dataItem.latestData; + units = item.dataItem.units; + decimals = item.dataItem.decimals; + } + if (!latestData) { + latestData = {} as FormattedData; + } + const value = formatFunction(item.param.value[1], latestData, units, decimals); + renderer.appendChild(valueElement, renderer.createText(value)); + renderer.setStyle(valueElement, 'flex', '1'); + renderer.setStyle(valueElement, 'text-align', 'end'); + renderer.setStyle(valueElement, 'font-family', settings.tooltipValueFont.family); + renderer.setStyle(valueElement, 'font-size', settings.tooltipValueFont.size + settings.tooltipValueFont.sizeUnit); + renderer.setStyle(valueElement, 'font-style', settings.tooltipValueFont.style); + renderer.setStyle(valueElement, 'font-weight', settings.tooltipValueFont.weight); + renderer.setStyle(valueElement, 'line-height', settings.tooltipValueFont.lineHeight); + renderer.setStyle(valueElement, 'color', settings.tooltipValueColor); + renderer.appendChild(labelValueElement, valueElement); + return labelValueElement; +}; + export enum TimeSeriesChartType { default = 'default', @@ -91,41 +470,6 @@ export const timeSeriesChartTypeTranslations = new Map( - [ - [SeriesFillType.none, 'widgets.time-series-chart.series.fill-type-none'], - [SeriesFillType.opacity, 'widgets.time-series-chart.series.fill-type-opacity'], - [SeriesFillType.gradient, 'widgets.time-series-chart.series.fill-type-gradient'] - ] -); - -export enum SeriesLabelPosition { - top = 'top', - bottom = 'bottom' -} - -export const seriesLabelPositions = Object.keys(SeriesLabelPosition) as SeriesLabelPosition[]; - -export const seriesLabelPositionTranslations = new Map( - [ - [SeriesLabelPosition.top, 'widgets.time-series-chart.series.label-position-top'], - [SeriesLabelPosition.bottom, 'widgets.time-series-chart.series.label-position-bottom'] - ] -); - export enum LineSeriesStepType { start = 'start', middle = 'middle', @@ -384,25 +698,25 @@ export const defaultTimeSeriesChartYAxisSettings: TimeSeriesChartYAxisSettings = weight: '600', lineHeight: '1' }, - labelColor: timeSeriesChartColorScheme['axis.label'].light, - position: AxisPosition.left, - showTickLabels: true, - tickLabelFont: { - family: 'Roboto', - size: 12, - sizeUnit: 'px', - style: 'normal', - weight: '400', - lineHeight: '1' + labelColor: chartColorScheme['axis.label'].light, + position: AxisPosition.left, + showTickLabels: true, + tickLabelFont: { + family: 'Roboto', + size: 12, + sizeUnit: 'px', + style: 'normal', + weight: '400', + lineHeight: '1' }, - tickLabelColor: timeSeriesChartColorScheme['axis.tickLabel'].light, - ticksFormatter: null, - showTicks: true, - ticksColor: timeSeriesChartColorScheme['axis.ticks'].light, - showLine: true, - lineColor: timeSeriesChartColorScheme['axis.line'].light, - showSplitLines: true, - splitLinesColor: timeSeriesChartColorScheme['axis.splitLine'].light + tickLabelColor: chartColorScheme['axis.tickLabel'].light, + ticksFormatter: null, + showTicks: true, + ticksColor: chartColorScheme['axis.ticks'].light, + showLine: true, + lineColor: chartColorScheme['axis.line'].light, + showSplitLines: true, + splitLinesColor: chartColorScheme['axis.splitLine'].light }; export const defaultTimeSeriesChartXAxisSettings: TimeSeriesChartXAxisSettings = { @@ -416,7 +730,7 @@ export const defaultTimeSeriesChartXAxisSettings: TimeSeriesChartXAxisSettings = weight: '600', lineHeight: '1' }, - labelColor: timeSeriesChartColorScheme['axis.label'].light, + labelColor: chartColorScheme['axis.label'].light, position: AxisPosition.bottom, showTickLabels: true, tickLabelFont: { @@ -427,14 +741,14 @@ export const defaultTimeSeriesChartXAxisSettings: TimeSeriesChartXAxisSettings = weight: '400', lineHeight: '1' }, - tickLabelColor: timeSeriesChartColorScheme['axis.tickLabel'].light, + tickLabelColor: chartColorScheme['axis.tickLabel'].light, ticksFormat: {}, showTicks: true, - ticksColor: timeSeriesChartColorScheme['axis.ticks'].light, + ticksColor: chartColorScheme['axis.ticks'].light, showLine: true, - lineColor: timeSeriesChartColorScheme['axis.line'].light, + lineColor: chartColorScheme['axis.line'].light, showSplitLines: true, - splitLinesColor: timeSeriesChartColorScheme['axis.splitLine'].light + splitLinesColor: chartColorScheme['axis.splitLine'].light }; export type TimeSeriesChartYAxes = {[id: TimeSeriesChartYAxisId]: TimeSeriesChartYAxisSettings}; @@ -453,9 +767,9 @@ export interface TimeSeriesChartThreshold { lineColor: string; lineType: TimeSeriesChartLineType | number | number[]; lineWidth: number; - startSymbol: EChartsShape; + startSymbol: ChartShape; startSymbolSize: number; - endSymbol: EChartsShape; + endSymbol: ChartShape; endSymbolSize: number; showLabel: boolean; labelPosition: ThresholdLabelPosition; @@ -505,12 +819,12 @@ export const timeSeriesChartThresholdDefaultSettings: TimeSeriesChartThreshold = yAxisId: 'default', units: null, decimals: 0, - lineColor: timeSeriesChartColorScheme['threshold.line'].light, + lineColor: chartColorScheme['threshold.line'].light, lineType: TimeSeriesChartLineType.solid, lineWidth: 1, - startSymbol: EChartsShape.none, + startSymbol: ChartShape.none, startSymbolSize: 5, - endSymbol: EChartsShape.arrow, + endSymbol: ChartShape.arrow, endSymbolSize: 5, showLabel: true, labelPosition: ThresholdLabelPosition.end, @@ -522,7 +836,7 @@ export const timeSeriesChartThresholdDefaultSettings: TimeSeriesChartThreshold = weight: '400', lineHeight: '1' }, - labelColor: timeSeriesChartColorScheme['threshold.label'].light, + labelColor: chartColorScheme['threshold.label'].light, enableLabelBackground: false, labelBackground: 'rgba(255,255,255,0.56)' }; @@ -657,7 +971,7 @@ export const timeSeriesChartGridDefaultSettings: TimeSeriesChartGridSettings = { borderColor: '#ccc' }; -export interface TimeSeriesChartSettings extends EChartsTooltipWidgetSettings, TimeSeriesChartComparisonSettings { +export interface TimeSeriesChartSettings extends TimeSeriesChartTooltipWidgetSettings, TimeSeriesChartComparisonSettings { thresholds: TimeSeriesChartThreshold[]; darkMode: boolean; dataZoom: boolean; @@ -665,7 +979,7 @@ export interface TimeSeriesChartSettings extends EChartsTooltipWidgetSettings, T grid: TimeSeriesChartGridSettings; yAxes: TimeSeriesChartYAxes; xAxis: TimeSeriesChartXAxisSettings; - animation: EChartsAnimationSettings; + animation: ChartAnimationSettings; barWidthSettings: TimeSeriesChartBarWidthSettings; noAggregationBarWidthSettings: TimeSeriesChartNoAggregationBarWidthSettings; visualMapSettings?: TimeSeriesChartVisualMapSettings; @@ -686,8 +1000,8 @@ export const timeSeriesChartDefaultSettings: TimeSeriesChartSettings = { }, xAxis: mergeDeep({} as TimeSeriesChartXAxisSettings, defaultTimeSeriesChartXAxisSettings), - animation: mergeDeep({} as EChartsAnimationSettings, - echartsAnimationDefaultSettings), + animation: mergeDeep({} as ChartAnimationSettings, + chartAnimationDefaultSettings), barWidthSettings: { barGap: 0.3, intervalGap: 0.6 @@ -695,7 +1009,7 @@ export const timeSeriesChartDefaultSettings: TimeSeriesChartSettings = { noAggregationBarWidthSettings: mergeDeep({} as TimeSeriesChartNoAggregationBarWidthSettings, timeSeriesChartNoAggregationBarWidthDefaultSettings), showTooltip: true, - tooltipTrigger: EChartsTooltipTrigger.axis, + tooltipTrigger: TimeSeriesChartTooltipTrigger.axis, tooltipLabelFont: { family: 'Roboto', size: 12, @@ -736,15 +1050,6 @@ export const timeSeriesChartDefaultSettings: TimeSeriesChartSettings = { { position: AxisPosition.top } as TimeSeriesChartXAxisSettings) }; -export interface SeriesFillSettings { - type: SeriesFillType; - opacity: number; - gradient: { - start: number; - end: number; - }; -} - export interface LineSeriesSettings { showLine: boolean; step: boolean; @@ -754,31 +1059,15 @@ export interface LineSeriesSettings { lineWidth: number; showPoints: boolean; showPointLabel: boolean; - pointLabelPosition: SeriesLabelPosition; + pointLabelPosition: ChartLabelPosition; pointLabelFont: Font; pointLabelColor: string; enablePointLabelBackground: boolean; pointLabelBackground: string; pointLabelFormatter?: string | LabelFormatterCallback; - pointShape: EChartsShape; + pointShape: ChartShape; pointSize: number; - fillAreaSettings: SeriesFillSettings; -} - -export interface BarSeriesSettings { - showBorder: boolean; - borderWidth: number; - borderRadius: number; - showLabel: boolean; - labelPosition: SeriesLabelPosition | BuiltinTextPosition; - labelFont: Font; - labelColor: string; - enableLabelBackground: boolean; - labelBackground: string; - labelFormatter?: string | LabelFormatterCallback; - labelLayout?: LabelLayoutOption | LabelLayoutOptionCallback; - additionalLabelOption?: {[key: string]: any}; - backgroundSettings: SeriesFillSettings; + fillAreaSettings: ChartFillSettings; } export interface TimeSeriesChartKeySettings extends DataKeySettingsWithComparison { @@ -787,8 +1076,8 @@ export interface TimeSeriesChartKeySettings extends DataKeySettingsWithCompariso dataHiddenByDefault: boolean; type: TimeSeriesChartSeriesType; lineSettings: LineSeriesSettings; - barSettings: BarSeriesSettings; - tooltipValueFormatter?: string | EChartsTooltipValueFormatFunction; + barSettings: ChartBarSettings; + tooltipValueFormatter?: string | TimeSeriesChartTooltipValueFormatFunction; } export const timeSeriesChartKeyDefaultSettings: TimeSeriesChartKeySettings = { @@ -805,7 +1094,7 @@ export const timeSeriesChartKeyDefaultSettings: TimeSeriesChartKeySettings = { lineWidth: 2, showPoints: false, showPointLabel: false, - pointLabelPosition: SeriesLabelPosition.top, + pointLabelPosition: ChartLabelPosition.top, pointLabelFont: { family: 'Roboto', size: 11, @@ -814,39 +1103,13 @@ export const timeSeriesChartKeyDefaultSettings: TimeSeriesChartKeySettings = { weight: '400', lineHeight: '1' }, - pointLabelColor: timeSeriesChartColorScheme['series.label'].light, + pointLabelColor: chartColorScheme['series.label'].light, enablePointLabelBackground: false, pointLabelBackground: 'rgba(255,255,255,0.56)', - pointShape: EChartsShape.emptyCircle, + pointShape: ChartShape.emptyCircle, pointSize: 4, fillAreaSettings: { - type: SeriesFillType.none, - opacity: 0.4, - gradient: { - start: 100, - end: 0 - } - } - }, - barSettings: { - showBorder: false, - borderWidth: 2, - borderRadius: 0, - showLabel: false, - labelPosition: SeriesLabelPosition.top, - labelFont: { - family: 'Roboto', - size: 11, - sizeUnit: 'px', - style: 'normal', - weight: '400', - lineHeight: '1' - }, - labelColor: timeSeriesChartColorScheme['series.label'].light, - enableLabelBackground: false, - labelBackground: 'rgba(255,255,255,0.56)', - backgroundSettings: { - type: SeriesFillType.none, + type: ChartFillType.none, opacity: 0.4, gradient: { start: 100, @@ -854,6 +1117,7 @@ export const timeSeriesChartKeyDefaultSettings: TimeSeriesChartKeySettings = { } } }, + barSettings: mergeDeep({} as ChartBarSettings, chartBarDefaultSettings), comparisonSettings: { showValuesForComparison: false, comparisonValuesLabel: '', @@ -861,14 +1125,6 @@ export const timeSeriesChartKeyDefaultSettings: TimeSeriesChartKeySettings = { } }; -export interface TimeSeriesChartDataItem extends EChartsSeriesItem { - xAxisIndex: number; - yAxisId: TimeSeriesChartYAxisId; - yAxisIndex: number; - option?: LineSeriesOption | CustomSeriesOption; - barRenderContext?: BarRenderContext; -} - type TimeSeriesChartThresholdValue = number | string | (number | string)[]; export interface TimeSeriesChartThresholdItem { @@ -1288,7 +1544,7 @@ const generateChartSeries = (dataItems: TimeSeriesChartDataItem[], return series; }; -export const updateDarkMode = (options: EChartsOption, settings: TimeSeriesChartSettings, +export const updateDarkMode = (options: EChartsOption, xAxisList: TimeSeriesChartXAxis[], yAxisList: TimeSeriesChartYAxis[], dataItems: TimeSeriesChartDataItem[], @@ -1332,7 +1588,7 @@ export const updateDarkMode = (options: EChartsOption, settings: TimeSeriesChart } } else { if (item.barRenderContext?.labelOption?.show) { - const barSettings = item.dataKey.settings as BarSeriesSettings; + const barSettings = item.dataKey.settings as ChartBarSettings; (item.barRenderContext.labelOption.rich.value as any).fill = prepareChartThemeColor(barSettings.labelColor, darkMode, 'series.label'); } @@ -1381,7 +1637,7 @@ const createTimeSeriesChartSeries = (item: TimeSeriesChartDataItem, lineSettings.pointLabelFont, lineSettings.pointLabelColor, lineSettings.enablePointLabelBackground, lineSettings.pointLabelBackground, lineSettings.pointLabelPosition, - lineSettings.pointLabelFormatter, false, darkMode); + lineSettings.pointLabelFormatter, false, darkMode) as SeriesLabelOption; lineSeriesOption.step = lineSettings.step ? lineSettings.stepType : false; lineSeriesOption.smooth = lineSettings.smooth ? 0.25 : false; if (lineSettings.smooth) { @@ -1391,11 +1647,11 @@ const createTimeSeriesChartSeries = (item: TimeSeriesChartDataItem, width: lineSettings.showLine ? lineSettings.lineWidth : 0, type: lineSettings.lineType }; - if (lineSettings.fillAreaSettings.type !== SeriesFillType.none) { + if (lineSettings.fillAreaSettings.type !== ChartFillType.none) { lineSeriesOption.areaStyle = {}; - if (lineSettings.fillAreaSettings.type === SeriesFillType.opacity) { + if (lineSettings.fillAreaSettings.type === ChartFillType.opacity) { lineSeriesOption.areaStyle.opacity = lineSettings.fillAreaSettings.opacity; - } else if (lineSettings.fillAreaSettings.type === SeriesFillType.gradient) { + } else if (lineSettings.fillAreaSettings.type === ChartFillType.gradient) { lineSeriesOption.areaStyle.opacity = 1; lineSeriesOption.areaStyle.color = createLinearOpacityGradient(seriesColor, lineSettings.fillAreaSettings.gradient); } @@ -1413,9 +1669,9 @@ const createTimeSeriesChartSeries = (item: TimeSeriesChartDataItem, borderWidth: barSettings.showBorder ? barSettings.borderWidth : 0, borderRadius: barSettings.borderRadius }; - if (barSettings.backgroundSettings.type === SeriesFillType.none) { + if (barSettings.backgroundSettings.type === ChartFillType.none) { barVisualSettings.color = seriesColor; - } else if (barSettings.backgroundSettings.type === SeriesFillType.opacity) { + } else if (barSettings.backgroundSettings.type === ChartFillType.opacity) { barVisualSettings.color = tinycolor(seriesColor).setAlpha(barSettings.backgroundSettings.opacity).toRgbString(); } else { barVisualSettings.color = createLinearOpacityGradient(seriesColor, barSettings.backgroundSettings.gradient); @@ -1437,10 +1693,10 @@ const createTimeSeriesChartSeries = (item: TimeSeriesChartDataItem, const createSeriesLabelOption = (item: TimeSeriesChartDataItem, show: boolean, labelFont: Font, labelColor: string, enableBackground: boolean, labelBackground: string, - position: SeriesLabelPosition | BuiltinTextPosition, + position: ChartLabelPosition | PieChartLabelPosition | BuiltinTextPosition, labelFormatter: string | LabelFormatterCallback, labelColorFill: boolean, - darkMode: boolean): SeriesLabelOption => { + darkMode: boolean): SeriesLabelOption | BarSeriesLabelOption => { let labelStyle: ComponentStyle = {}; if (show) { labelStyle = createChartTextStyle(labelFont, labelColor, darkMode, 'series.label', labelColorFill); @@ -1467,7 +1723,7 @@ const createSeriesLabelOption = (item: TimeSeriesChartDataItem, show: boolean, } return result; }; - const labelOption: SeriesLabelOption = { + const labelOption: SeriesLabelOption | BarSeriesLabelOption = { show, position, formatter, @@ -1482,45 +1738,3 @@ const createSeriesLabelOption = (item: TimeSeriesChartDataItem, show: boolean, } return labelOption; }; - -const createChartTextStyle = (font: Font, color: string, darkMode: boolean, colorKey?: string, fill = false): ComponentStyle => { - const style = textStyle(font); - delete style.lineHeight; - style.fontSize = font.size; - if (fill) { - style.fill = prepareChartThemeColor(color, darkMode, colorKey); - } else { - style.color = prepareChartThemeColor(color, darkMode, colorKey); - } - return style; -}; - -const createLinearOpacityGradient = (color: string, gradient: {start: number; end: number}): LinearGradientObject => ({ - type: 'linear', - x: 0, - y: 0, - x2: 0, - y2: 1, - colorStops: [{ - offset: 0, color: tinycolor(color).setAlpha(gradient.start / 100).toRgbString() // color at 0% - }, { - offset: 1, color: tinycolor(color).setAlpha(gradient.end / 100).toRgbString() // color at 100% - }], - global: false -}); - -const prepareChartThemeColor = (color: string, darkMode: boolean, colorKey?: string): string => { - if (darkMode) { - let colorInstance = tinycolor(color); - if (colorInstance.isDark()) { - if (colorKey && timeSeriesChartColorScheme[colorKey]) { - return timeSeriesChartColorScheme[colorKey].dark; - } else { - const rgb = colorInstance.toRgb(); - colorInstance = tinycolor({r: 255 - rgb.r, g: 255 - rgb.g, b: 255 - rgb.b, a: rgb.a}); - return colorInstance.toRgbString(); - } - } - } - return color; -}; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts index ad5aafd4eb..e1d9ca2324 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/chart/time-series-chart.ts @@ -16,15 +16,16 @@ import { WidgetContext } from '@home/models/widget-component.models'; import { + adjustTimeAxisExtentToData, calculateThresholdsOffset, createTimeSeriesVisualMapOption, createTimeSeriesXAxis, createTimeSeriesYAxis, + createTooltipValueFormatFunction, defaultTimeSeriesChartYAxisSettings, generateChartData, LineSeriesStepType, parseThresholdData, - SeriesLabelPosition, TimeSeriesChartAxis, TimeSeriesChartDataItem, timeSeriesChartDefaultSettings, @@ -37,30 +38,27 @@ import { timeSeriesChartThresholdDefaultSettings, TimeSeriesChartThresholdItem, TimeSeriesChartThresholdType, + timeSeriesChartTooltipFormatter, + TimeSeriesChartTooltipTrigger, + TimeSeriesChartTooltipValueFormatFunction, TimeSeriesChartType, TimeSeriesChartXAxis, TimeSeriesChartYAxis, TimeSeriesChartYAxisId, TimeSeriesChartYAxisSettings, + toTimeSeriesChartDataSet, updateDarkMode, updateXAxisTimeWindow } from '@home/components/widget/lib/chart/time-series-chart.models'; import { ResizeObserver } from '@juggle/resize-observer'; import { - adjustTimeAxisExtentToData, calculateAxisSize, - createTooltipValueFormatFunction, ECharts, echartsModule, EChartsOption, - EChartsShape, - echartsTooltipFormatter, - EChartsTooltipTrigger, - EChartsTooltipValueFormatFunction, getAxisExtent, getFocusedSeriesIndex, - measureAxisNameSize, - toNamedData + measureAxisNameSize } from '@home/components/widget/lib/chart/echarts-widget.models'; import { DateFormatProcessor } from '@shared/models/widget-settings.models'; import { formattedDataFormDatasourceData, formatValue, isDefinedAndNotNull, isEqual, mergeDeep } from '@core/utils'; @@ -76,6 +74,7 @@ import { DataKeySettingsFunction } from '@home/components/widget/config/data-key import { DeepPartial } from '@shared/models/common'; import { BarRenderSharedContext } from '@home/components/widget/lib/chart/time-series-chart-bar.models'; import { TimeSeriesChartStateValueConverter } from '@home/components/widget/lib/chart/time-series-chart-state.models'; +import { ChartLabelPosition, ChartShape } from '@home/components/widget/lib/chart/chart.models'; export class TbTimeSeriesChart { @@ -92,14 +91,14 @@ export class TbTimeSeriesChart { settings.type = TimeSeriesChartSeriesType.line; settings.lineSettings.showLine = false; settings.lineSettings.showPoints = true; - settings.lineSettings.pointShape = EChartsShape.circle; + settings.lineSettings.pointShape = ChartShape.circle; settings.lineSettings.pointSize = 8; } else if (type === TimeSeriesChartType.state) { settings.type = TimeSeriesChartSeriesType.line; settings.lineSettings.showLine = true; settings.lineSettings.step = true; settings.lineSettings.stepType = LineSeriesStepType.end; - settings.lineSettings.pointShape = EChartsShape.circle; + settings.lineSettings.pointShape = ChartShape.circle; settings.lineSettings.pointSize = 12; } return settings; @@ -135,7 +134,7 @@ export class TbTimeSeriesChart { private timeSeriesChartOptions: EChartsOption; private readonly tooltipDateFormat: DateFormatProcessor; - private readonly tooltipValueFormatFunction: EChartsTooltipValueFormatFunction; + private readonly tooltipValueFormatFunction: TimeSeriesChartTooltipValueFormatFunction; private readonly stateValueConverter: TimeSeriesChartStateValueConverter; private yMinSubject = new BehaviorSubject(-1); @@ -189,7 +188,7 @@ export class TbTimeSeriesChart { this.tooltipValueFormatFunction = createTooltipValueFormatFunction(this.settings.tooltipValueFormatter); if (!this.tooltipValueFormatFunction) { - this.tooltipValueFormatFunction = (value, latestData, units, decimals) => formatValue(value, decimals, units, false); + this.tooltipValueFormatFunction = (value, _latestData, units, decimals) => formatValue(value, decimals, units, false); } } } @@ -218,7 +217,7 @@ export class TbTimeSeriesChart { const datasourceData = this.ctx.data ? this.ctx.data.find(d => d.dataKey === item.dataKey) : null; if (!isEqual(item.dataSet, datasourceData?.data)) { item.dataSet = datasourceData?.data; - item.data = datasourceData?.data ? toNamedData(datasourceData.data, this.stateValueConverter?.valueConverter) : []; + item.data = datasourceData?.data ? toTimeSeriesChartDataSet(datasourceData.data, this.stateValueConverter?.valueConverter) : []; } } this.onResize(); @@ -352,7 +351,7 @@ export class TbTimeSeriesChart { this.darkMode = darkMode; if (this.timeSeriesChart) { this.timeSeriesChartOptions = updateDarkMode(this.timeSeriesChartOptions, - this.settings, this.xAxisList, this.yAxisList, this.dataItems, darkMode); + this.xAxisList, this.yAxisList, this.dataItems, darkMode); this.timeSeriesChart.setOption(this.timeSeriesChartOptions); } } @@ -381,11 +380,11 @@ export class TbTimeSeriesChart { const keySettings = mergeDeep({} as TimeSeriesChartKeySettings, timeSeriesChartKeyDefaultSettings, dataKey.settings); if ((keySettings.type === TimeSeriesChartSeriesType.line && keySettings.lineSettings.showPointLabel && - keySettings.lineSettings.pointLabelPosition === SeriesLabelPosition.top) || + keySettings.lineSettings.pointLabelPosition === ChartLabelPosition.top) || (keySettings.type === TimeSeriesChartSeriesType.bar && keySettings.barSettings.showLabel && - [SeriesLabelPosition.top, SeriesLabelPosition.bottom] - .includes(keySettings.barSettings.labelPosition as SeriesLabelPosition))) { + [ChartLabelPosition.top, ChartLabelPosition.bottom] + .includes(keySettings.barSettings.labelPosition as ChartLabelPosition))) { this.topPointLabels = true; } if (this.stateValueConverter && keySettings.type === TimeSeriesChartSeriesType.line) { @@ -393,7 +392,8 @@ export class TbTimeSeriesChart { } dataKey.settings = keySettings; const datasourceData = this.ctx.data ? this.ctx.data.find(d => d.dataKey === dataKey) : null; - const namedData = datasourceData?.data ? toNamedData(datasourceData.data, this.stateValueConverter?.valueConverter) : []; + const data = datasourceData?.data ? + toTimeSeriesChartDataSet(datasourceData.data, this.stateValueConverter?.valueConverter) : []; const units = dataKey.units && dataKey.units.length ? dataKey.units : this.ctx.units; const decimals = isDefinedAndNotNull(dataKey.decimals) ? dataKey.decimals : (isDefinedAndNotNull(this.ctx.decimals) ? this.ctx.decimals : 2); @@ -413,7 +413,7 @@ export class TbTimeSeriesChart { comparisonItem, datasource, dataKey, - data: namedData, + data, enabled: !keySettings.dataHiddenByDefault, tooltipValueFormatFunction: createTooltipValueFormatFunction(keySettings.tooltipValueFormatter) }); @@ -596,14 +596,14 @@ export class TbTimeSeriesChart { darkMode: this.darkMode, backgroundColor: 'transparent', tooltip: [{ - trigger: this.settings.tooltipTrigger === EChartsTooltipTrigger.axis ? 'axis' : 'item', + trigger: this.settings.tooltipTrigger === TimeSeriesChartTooltipTrigger.axis ? 'axis' : 'item', confine: true, appendTo: 'body', axisPointer: { type: this.noAggregation ? 'line' : 'shadow' }, formatter: (params: CallbackDataParams[]) => - this.settings.showTooltip ? echartsTooltipFormatter(this.renderer, this.tooltipDateFormat, + this.settings.showTooltip ? timeSeriesChartTooltipFormatter(this.renderer, this.tooltipDateFormat, this.settings, params, this.tooltipValueFormatFunction, this.settings.tooltipShowFocusedSeries ? getFocusedSeriesIndex(this.timeSeriesChart) : -1, this.dataItems, this.noAggregation ? null : this.ctx.timeWindow.interval) : undefined, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.html new file mode 100644 index 0000000000..36ff5a4d0d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.html @@ -0,0 +1,169 @@ + + +
+
widgets.bar-chart.bar-chart-style
+
+ + {{ 'widgets.latest-chart.sort-series' | translate }} + +
+
+
widgets.bar-chart.bar-appearance
+ + +
+
+
widgets.bar-chart.bar-axis
+
+
widgets.chart.chart-axis.scale
+
+
widgets.chart.chart-axis.scale-min
+ + + +
widgets.chart.chart-axis.scale-max
+ + + + + + + +
+
+
+
+ + + + + {{ 'widget-config.legend' | translate }} + + + + +
+
{{ 'legend.position' | translate }}
+ + + + {{ legendPositionTranslationMap.get(pos) | translate }} + + + +
+
+
{{ 'legend.label' | translate }}
+
+ + + + +
+
+
+
{{ 'legend.value' | translate }}
+
+ + + + +
+
+ + +
+
+ + + + + {{ 'widget-config.tooltip' | translate }} + + + + +
+
{{ 'tooltip.value' | translate }}
+
+ + + + {{ latestChartTooltipValueTypeTranslationMap.get(type) | translate }} + + + + + +
widget-config.decimals-suffix
+ + + + + +
+
+
+
{{ 'tooltip.background-color' | translate }}
+ + +
+
+
{{ 'tooltip.background-blur' | translate }}
+ + +
px
+ +
+ + +
+ + +
+
{{ 'widgets.background.background' | translate }}
+ + +
+
+ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.ts new file mode 100644 index 0000000000..2074382257 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-widget-settings.component.ts @@ -0,0 +1,160 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { + legendPositions, + legendPositionTranslationMap, + WidgetSettings, + WidgetSettingsComponent +} from '@shared/models/widget.models'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { formatValue, mergeDeep } from '@core/utils'; +import { + LatestChartTooltipValueType, + latestChartTooltipValueTypes, + latestChartTooltipValueTypeTranslations +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { + barChartWidgetDefaultSettings, + BarChartWidgetSettings +} from '@home/components/widget/lib/chart/bar-chart-widget.models'; + +@Component({ + selector: 'tb-bar-chart-widget-settings', + templateUrl: './bar-chart-widget-settings.component.html', + styleUrls: [] +}) +export class BarChartWidgetSettingsComponent extends WidgetSettingsComponent { + + legendPositions = legendPositions; + + legendPositionTranslationMap = legendPositionTranslationMap; + + latestChartTooltipValueTypes = latestChartTooltipValueTypes; + + latestChartTooltipValueTypeTranslationMap = latestChartTooltipValueTypeTranslations; + + barChartWidgetSettingsForm: UntypedFormGroup; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + tooltipValuePreviewFn = this._tooltipValuePreviewFn.bind(this); + + constructor(protected store: Store, + private fb: UntypedFormBuilder) { + super(store); + } + + protected settingsForm(): UntypedFormGroup { + return this.barChartWidgetSettingsForm; + } + + protected defaultSettings(): WidgetSettings { + return mergeDeep({} as BarChartWidgetSettings, barChartWidgetDefaultSettings); + } + + protected onSettingsSet(settings: WidgetSettings) { + this.barChartWidgetSettingsForm = this.fb.group({ + + sortSeries: [settings.sortSeries, []], + + barSettings: [settings.barSettings, []], + + axisMin: [settings.axisMin, []], + axisMax: [settings.axisMax, []], + axisTickLabelFont: [settings.axisTickLabelFont, []], + axisTickLabelColor: [settings.axisTickLabelColor, []], + + animation: [settings.animation, []], + + showLegend: [settings.showLegend, []], + legendPosition: [settings.legendPosition, []], + legendLabelFont: [settings.legendLabelFont, []], + legendLabelColor: [settings.legendLabelColor, []], + legendValueFont: [settings.legendValueFont, []], + legendValueColor: [settings.legendValueColor, []], + + showTooltip: [settings.showTooltip, []], + tooltipValueType: [settings.tooltipValueType, []], + tooltipValueDecimals: [settings.tooltipValueDecimals, []], + tooltipValueFont: [settings.tooltipValueFont, []], + tooltipValueColor: [settings.tooltipValueColor, []], + tooltipBackgroundColor: [settings.tooltipBackgroundColor, []], + tooltipBackgroundBlur: [settings.tooltipBackgroundBlur, []], + + background: [settings.background, []] + }); + } + + protected validatorTriggers(): string[] { + return ['showLegend', 'showTooltip']; + } + + protected updateValidators(emitEvent: boolean) { + const showLegend: boolean = this.barChartWidgetSettingsForm.get('showLegend').value; + const showTooltip: boolean = this.barChartWidgetSettingsForm.get('showTooltip').value; + + if (showLegend) { + this.barChartWidgetSettingsForm.get('legendPosition').enable(); + this.barChartWidgetSettingsForm.get('legendLabelFont').enable(); + this.barChartWidgetSettingsForm.get('legendLabelColor').enable(); + this.barChartWidgetSettingsForm.get('legendValueFont').enable(); + this.barChartWidgetSettingsForm.get('legendValueColor').enable(); + } else { + this.barChartWidgetSettingsForm.get('legendPosition').disable(); + this.barChartWidgetSettingsForm.get('legendLabelFont').disable(); + this.barChartWidgetSettingsForm.get('legendLabelColor').disable(); + this.barChartWidgetSettingsForm.get('legendValueFont').disable(); + this.barChartWidgetSettingsForm.get('legendValueColor').disable(); + } + if (showTooltip) { + this.barChartWidgetSettingsForm.get('tooltipValueType').enable(); + this.barChartWidgetSettingsForm.get('tooltipValueDecimals').enable(); + this.barChartWidgetSettingsForm.get('tooltipValueFont').enable(); + this.barChartWidgetSettingsForm.get('tooltipValueColor').enable(); + this.barChartWidgetSettingsForm.get('tooltipBackgroundColor').enable(); + this.barChartWidgetSettingsForm.get('tooltipBackgroundBlur').enable(); + } else { + this.barChartWidgetSettingsForm.get('tooltipValueType').disable(); + this.barChartWidgetSettingsForm.get('tooltipValueDecimals').disable(); + this.barChartWidgetSettingsForm.get('tooltipValueFont').disable(); + this.barChartWidgetSettingsForm.get('tooltipValueColor').disable(); + this.barChartWidgetSettingsForm.get('tooltipBackgroundColor').disable(); + this.barChartWidgetSettingsForm.get('tooltipBackgroundBlur').disable(); + } + } + + private _valuePreviewFn(): string { + const units: string = this.widgetConfig.config.units; + const decimals: number = this.widgetConfig.config.decimals; + return formatValue(110, decimals, units, false); + } + + private _tooltipValuePreviewFn(): string { + const tooltipValueType: LatestChartTooltipValueType = this.barChartWidgetSettingsForm.get('tooltipValueType').value; + const decimals: number = this.barChartWidgetSettingsForm.get('tooltipValueDecimals').value; + if (tooltipValueType === LatestChartTooltipValueType.percentage) { + return formatValue(35, decimals, '%', false); + } else { + const units: string = this.widgetConfig.config.units; + return formatValue(110, decimals, units, false); + } + } + +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.html index 920374ec00..8965b462b1 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/bar-chart-with-labels-widget-settings.component.html @@ -62,26 +62,26 @@
- {{ 'widgets.time-series-chart.series.bar.show-border' | translate }} + {{ 'widgets.chart.bar.show-border' | translate }}
-
widgets.time-series-chart.series.bar.border-width
+
widgets.chart.bar.border-width
-
widgets.time-series-chart.series.bar.border-radius
+
widgets.chart.bar.border-radius
- - + title="widgets.chart.background" + fillNoneTitle="widgets.chart.fill-type-solid"> + @@ -227,9 +227,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/doughnut-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/doughnut-widget-settings.component.html index 863663db08..4f747206a0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/doughnut-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/doughnut-widget-settings.component.html @@ -154,9 +154,9 @@
- - +
{{ 'widgets.background.background' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.html index ea96d8a4b5..9674bb8d6f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.html @@ -172,9 +172,9 @@
- - +
{{ 'widgets.background.background' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.ts index ae96753264..19b49247cd 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/pie-chart-widget-settings.component.ts @@ -33,7 +33,7 @@ import { import { pieChartLabelPositions, pieChartLabelPositionTranslations -} from '@home/components/widget/lib/chart/pie-chart.models'; +} from '@home/components/widget/lib/chart/chart.models'; import { pieChartWidgetDefaultSettings, PieChartWidgetSettings diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.html new file mode 100644 index 0000000000..9201173d4d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.html @@ -0,0 +1,176 @@ + + +
+
widgets.polar-area-chart.polar-area-chart-style
+
+ + {{ 'widgets.latest-chart.sort-series' | translate }} + +
+
+
widgets.bar-chart.bar-appearance
+ + +
+
+
widgets.polar-area-chart.polar-axis
+
+
widgets.chart.chart-axis.scale
+
+
widgets.chart.chart-axis.scale-min
+ + + +
widgets.chart.chart-axis.scale-max
+ + + + + + + +
+
+
+
widgets.polar-area-chart.start-angle
+ + + +
+
+
+ + + + + {{ 'widget-config.legend' | translate }} + + + + +
+
{{ 'legend.position' | translate }}
+ + + + {{ legendPositionTranslationMap.get(pos) | translate }} + + + +
+
+
{{ 'legend.label' | translate }}
+
+ + + + +
+
+
+
{{ 'legend.value' | translate }}
+
+ + + + +
+
+ + +
+
+ + + + + {{ 'widget-config.tooltip' | translate }} + + + + +
+
{{ 'tooltip.value' | translate }}
+
+ + + + {{ latestChartTooltipValueTypeTranslationMap.get(type) | translate }} + + + + + +
widget-config.decimals-suffix
+ + + + + +
+
+
+
{{ 'tooltip.background-color' | translate }}
+ + +
+
+
{{ 'tooltip.background-blur' | translate }}
+ + +
px
+ +
+ + +
+ + +
+
{{ 'widgets.background.background' | translate }}
+ + +
+
+ diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.ts new file mode 100644 index 0000000000..64144b5319 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component.ts @@ -0,0 +1,161 @@ +/// +/// Copyright © 2016-2024 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component } from '@angular/core'; +import { + legendPositions, + legendPositionTranslationMap, + WidgetSettings, + WidgetSettingsComponent +} from '@shared/models/widget.models'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { formatValue, mergeDeep } from '@core/utils'; +import { + LatestChartTooltipValueType, + latestChartTooltipValueTypes, + latestChartTooltipValueTypeTranslations +} from '@home/components/widget/lib/chart/latest-chart.models'; +import { + polarAreaChartWidgetDefaultSettings, + PolarAreaChartWidgetSettings +} from '@home/components/widget/lib/chart/polar-area-widget.models'; + +@Component({ + selector: 'tb-polar-area-chart-widget-settings', + templateUrl: './polar-area-chart-widget-settings.component.html', + styleUrls: [] +}) +export class PolarAreaChartWidgetSettingsComponent extends WidgetSettingsComponent { + + legendPositions = legendPositions; + + legendPositionTranslationMap = legendPositionTranslationMap; + + latestChartTooltipValueTypes = latestChartTooltipValueTypes; + + latestChartTooltipValueTypeTranslationMap = latestChartTooltipValueTypeTranslations; + + polarAreaChartWidgetSettingsForm: UntypedFormGroup; + + valuePreviewFn = this._valuePreviewFn.bind(this); + + tooltipValuePreviewFn = this._tooltipValuePreviewFn.bind(this); + + constructor(protected store: Store, + private fb: UntypedFormBuilder) { + super(store); + } + + protected settingsForm(): UntypedFormGroup { + return this.polarAreaChartWidgetSettingsForm; + } + + protected defaultSettings(): WidgetSettings { + return mergeDeep({} as PolarAreaChartWidgetSettings, polarAreaChartWidgetDefaultSettings); + } + + protected onSettingsSet(settings: WidgetSettings) { + this.polarAreaChartWidgetSettingsForm = this.fb.group({ + + sortSeries: [settings.sortSeries, []], + + barSettings: [settings.barSettings, []], + + axisMin: [settings.axisMin, []], + axisMax: [settings.axisMax, []], + axisTickLabelFont: [settings.axisTickLabelFont, []], + axisTickLabelColor: [settings.axisTickLabelColor, []], + angleAxisStartAngle: [settings.angleAxisStartAngle, [Validators.min(0), Validators.max(360)]], + + animation: [settings.animation, []], + + showLegend: [settings.showLegend, []], + legendPosition: [settings.legendPosition, []], + legendLabelFont: [settings.legendLabelFont, []], + legendLabelColor: [settings.legendLabelColor, []], + legendValueFont: [settings.legendValueFont, []], + legendValueColor: [settings.legendValueColor, []], + + showTooltip: [settings.showTooltip, []], + tooltipValueType: [settings.tooltipValueType, []], + tooltipValueDecimals: [settings.tooltipValueDecimals, []], + tooltipValueFont: [settings.tooltipValueFont, []], + tooltipValueColor: [settings.tooltipValueColor, []], + tooltipBackgroundColor: [settings.tooltipBackgroundColor, []], + tooltipBackgroundBlur: [settings.tooltipBackgroundBlur, []], + + background: [settings.background, []] + }); + } + + protected validatorTriggers(): string[] { + return ['showLegend', 'showTooltip']; + } + + protected updateValidators(emitEvent: boolean) { + const showLegend: boolean = this.polarAreaChartWidgetSettingsForm.get('showLegend').value; + const showTooltip: boolean = this.polarAreaChartWidgetSettingsForm.get('showTooltip').value; + + if (showLegend) { + this.polarAreaChartWidgetSettingsForm.get('legendPosition').enable(); + this.polarAreaChartWidgetSettingsForm.get('legendLabelFont').enable(); + this.polarAreaChartWidgetSettingsForm.get('legendLabelColor').enable(); + this.polarAreaChartWidgetSettingsForm.get('legendValueFont').enable(); + this.polarAreaChartWidgetSettingsForm.get('legendValueColor').enable(); + } else { + this.polarAreaChartWidgetSettingsForm.get('legendPosition').disable(); + this.polarAreaChartWidgetSettingsForm.get('legendLabelFont').disable(); + this.polarAreaChartWidgetSettingsForm.get('legendLabelColor').disable(); + this.polarAreaChartWidgetSettingsForm.get('legendValueFont').disable(); + this.polarAreaChartWidgetSettingsForm.get('legendValueColor').disable(); + } + if (showTooltip) { + this.polarAreaChartWidgetSettingsForm.get('tooltipValueType').enable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueDecimals').enable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueFont').enable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueColor').enable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipBackgroundColor').enable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipBackgroundBlur').enable(); + } else { + this.polarAreaChartWidgetSettingsForm.get('tooltipValueType').disable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueDecimals').disable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueFont').disable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipValueColor').disable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipBackgroundColor').disable(); + this.polarAreaChartWidgetSettingsForm.get('tooltipBackgroundBlur').disable(); + } + } + + private _valuePreviewFn(): string { + const units: string = this.widgetConfig.config.units; + const decimals: number = this.widgetConfig.config.decimals; + return formatValue(110, decimals, units, false); + } + + private _tooltipValuePreviewFn(): string { + const tooltipValueType: LatestChartTooltipValueType = this.polarAreaChartWidgetSettingsForm.get('tooltipValueType').value; + const decimals: number = this.polarAreaChartWidgetSettingsForm.get('tooltipValueDecimals').value; + if (tooltipValueType === LatestChartTooltipValueType.percentage) { + return formatValue(35, decimals, '%', false); + } else { + const units: string = this.widgetConfig.config.units; + return formatValue(110, decimals, units, false); + } + } + +} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.html index 30c5eea4c1..87bbe59e1f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.html @@ -121,8 +121,8 @@
- - {{ seriesLabelPositionTranslations.get(position) | translate }} + + {{ chartLabelPositionTranslations.get(position) | translate }} @@ -151,8 +151,8 @@
widgets.time-series-chart.series.point.point-shape
- - {{ echartsShapeTranslations.get(shape) | translate }} + + {{ chartShapeTranslations.get(shape) | translate }} @@ -303,9 +303,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts index 872a77d2a7..895fc6888a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/range-chart-widget-settings.component.ts @@ -34,12 +34,10 @@ import { DateFormatProcessor, DateFormatSettings } from '@shared/models/widget-s import { lineSeriesStepTypes, lineSeriesStepTypeTranslations, - seriesLabelPositions, - seriesLabelPositionTranslations, timeSeriesLineTypes, timeSeriesLineTypeTranslations } from '@home/components/widget/lib/chart/time-series-chart.models'; -import { echartsShapes, echartsShapeTranslations } from '@home/components/widget/lib/chart/echarts-widget.models'; +import { chartLabelPositions, chartLabelPositionTranslations, chartShapes, chartShapeTranslations } from '@home/components/widget/lib/chart/chart.models'; @Component({ selector: 'tb-range-chart-widget-settings', @@ -65,13 +63,13 @@ export class RangeChartWidgetSettingsComponent extends WidgetSettingsComponent { timeSeriesLineTypeTranslations = timeSeriesLineTypeTranslations; - seriesLabelPositions = seriesLabelPositions; + chartLabelPositions = chartLabelPositions; - seriesLabelPositionTranslations = seriesLabelPositionTranslations; + chartLabelPositionTranslations = chartLabelPositionTranslations; - echartsShapes = echartsShapes; + chartShapes = chartShapes; - echartsShapeTranslations = echartsShapeTranslations; + chartShapeTranslations = chartShapeTranslations; legendPositions = legendPositions; @@ -164,7 +162,7 @@ export class RangeChartWidgetSettingsComponent extends WidgetSettingsComponent { 'showLegend', 'showTooltip', 'tooltipShowDate']; } - protected updateValidators(emitEvent: boolean) { + protected updateValidators() { const showRangeThresholds: boolean = this.rangeChartWidgetSettingsForm.get('showRangeThresholds').value; const fillArea: boolean = this.rangeChartWidgetSettingsForm.get('fillArea').value; const showLine: boolean = this.rangeChartWidgetSettingsForm.get('showLine').value; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-key-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-key-settings.component.html index a265a1b431..1299f0574a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-key-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-key-settings.component.html @@ -52,9 +52,9 @@ [chartType]="chartType" formControlName="lineSettings"> - - + +
widgets.chart.tooltip-settings
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.html index f20f1323b3..906c4148a2 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.html @@ -66,9 +66,9 @@
- - +
@@ -80,8 +80,8 @@
- - {{ seriesLabelPositionTranslations.get(position) | translate }} + + {{ chartLabelPositionTranslations.get(position) | translate }} @@ -110,8 +110,8 @@
widgets.time-series-chart.series.point.point-shape
- - {{ echartsShapeTranslations.get(shape) | translate }} + + {{ chartShapeTranslations.get(shape) | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.ts index 1d57220b4e..4e970a5198 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-line-settings.component.ts @@ -26,18 +26,21 @@ import { LineSeriesSettings, lineSeriesStepTypes, lineSeriesStepTypeTranslations, - seriesLabelPositions, - seriesLabelPositionTranslations, TimeSeriesChartType, timeSeriesLineTypes, timeSeriesLineTypeTranslations } from '@home/components/widget/lib/chart/time-series-chart.models'; -import { echartsShapes, echartsShapeTranslations } from '@home/components/widget/lib/chart/echarts-widget.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { merge } from 'rxjs'; import { formatValue, isDefinedAndNotNull } from '@core/utils'; import { DataKeyConfigComponent } from '@home/components/widget/config/data-key-config.component'; +import { + chartLabelPositions, + chartLabelPositionTranslations, + chartShapes, + chartShapeTranslations +} from '@home/components/widget/lib/chart/chart.models'; @Component({ selector: 'tb-time-series-chart-line-settings', @@ -63,13 +66,13 @@ export class TimeSeriesChartLineSettingsComponent implements OnInit, ControlValu timeSeriesLineTypeTranslations = timeSeriesLineTypeTranslations; - seriesLabelPositions = seriesLabelPositions; + chartLabelPositions = chartLabelPositions; - seriesLabelPositionTranslations = seriesLabelPositionTranslations; + chartLabelPositionTranslations = chartLabelPositionTranslations; - echartsShapes = echartsShapes; + chartShapes = chartShapes; - echartsShapeTranslations = echartsShapeTranslations; + chartShapeTranslations = chartShapeTranslations; pointLabelPreviewFn = this._pointLabelPreviewFn.bind(this); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.html index 6bf8abdb70..79a517a7d8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.html @@ -254,9 +254,9 @@
- - +
widget-config.card-appearance
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts index 8ea641d299..32d3f5dd80 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component.ts @@ -28,12 +28,12 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { formatValue, isDefinedAndNotNull, mergeDeep } from '@core/utils'; import { DateFormatProcessor, DateFormatSettings } from '@shared/models/widget-settings.models'; -import { EChartsTooltipTrigger } from '../../chart/echarts-widget.models'; import { timeSeriesChartWidgetDefaultSettings, TimeSeriesChartWidgetSettings } from '@home/components/widget/lib/chart/time-series-chart-widget.models'; import { + TimeSeriesChartTooltipTrigger, TimeSeriesChartKeySettings, TimeSeriesChartType, TimeSeriesChartYAxes, @@ -65,7 +65,7 @@ export class TimeSeriesChartWidgetSettingsComponent extends WidgetSettingsCompon TimeSeriesChartType = TimeSeriesChartType; - EChartsTooltipTrigger = EChartsTooltipTrigger; + EChartsTooltipTrigger = TimeSeriesChartTooltipTrigger; legendPositions = legendPositions; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.html similarity index 82% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.html rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.html index 53246d2cfc..b0a091a5bc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.html @@ -23,61 +23,61 @@ - {{ 'widgets.echarts.animation.animation' | translate }} + {{ 'widgets.chart.animation.animation' | translate }}
-
{{ 'widgets.echarts.animation.animation-threshold' | translate }}
+
{{ 'widgets.chart.animation.animation-threshold' | translate }}
ms
-
{{ 'widgets.echarts.animation.animation-duration' | translate }}
+
{{ 'widgets.chart.animation.animation-duration' | translate }}
ms
-
widgets.echarts.animation.animation-easing
+
widgets.chart.animation.animation-easing
- + {{ easing }}
-
{{ 'widgets.echarts.animation.animation-delay' | translate }}
+
{{ 'widgets.chart.animation.animation-delay' | translate }}
ms
-
{{ 'widgets.echarts.animation.update-animation-duration' | translate }}
+
{{ 'widgets.chart.animation.update-animation-duration' | translate }}
ms
-
widgets.echarts.animation.update-animation-easing
+
widgets.chart.animation.update-animation-easing
- + {{ easing }}
-
{{ 'widgets.echarts.animation.update-animation-delay' | translate }}
+
{{ 'widgets.chart.animation.update-animation-delay' | translate }}
ms
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.ts similarity index 81% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.ts rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.ts index b3a10c6298..19ccfcd18b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/echarts-animation-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-animation-settings.component.ts @@ -22,41 +22,36 @@ import { UntypedFormGroup, Validators } from '@angular/forms'; -import { WidgetService } from '@core/http/widget.service'; -import { - echartsAnimationEasings, - EChartsAnimationSettings -} from '@home/components/widget/lib/chart/echarts-widget.models'; +import { chartAnimationEasings, ChartAnimationSettings } from '@home/components/widget/lib/chart/chart.models'; @Component({ - selector: 'tb-echarts-animation-settings', - templateUrl: './echarts-animation-settings.component.html', + selector: 'tb-chart-animation-settings', + templateUrl: './chart-animation-settings.component.html', styleUrls: ['./../../widget-settings.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => EchartsAnimationSettingsComponent), + useExisting: forwardRef(() => ChartAnimationSettingsComponent), multi: true } ] }) -export class EchartsAnimationSettingsComponent implements OnInit, ControlValueAccessor { +export class ChartAnimationSettingsComponent implements OnInit, ControlValueAccessor { settingsExpanded = false; - echartsAnimationEasings = echartsAnimationEasings; + chartAnimationEasings = chartAnimationEasings; @Input() disabled: boolean; - private modelValue: EChartsAnimationSettings; + private modelValue: ChartAnimationSettings; private propagateChange = null; public animationSettingsFormGroup: UntypedFormGroup; - constructor(private fb: UntypedFormBuilder, - private widgetService: WidgetService,) { + constructor(private fb: UntypedFormBuilder) { } ngOnInit(): void { @@ -96,7 +91,7 @@ export class EchartsAnimationSettingsComponent implements OnInit, ControlValueAc } } - writeValue(value: EChartsAnimationSettings): void { + writeValue(value: ChartAnimationSettings): void { this.modelValue = value; this.animationSettingsFormGroup.patchValue( value, {emitEvent: false} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.html similarity index 70% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.html rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.html index ddbeaf2480..07fab8b7ce 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.html @@ -18,32 +18,39 @@
- {{ 'widgets.time-series-chart.series.bar.show-border' | translate }} + {{ 'widgets.chart.bar.show-border' | translate }}
-
widgets.time-series-chart.series.bar.border-width
+
widgets.chart.bar.border-width
-
widgets.time-series-chart.series.bar.border-radius
+
widgets.chart.bar.border-radius
+
+
widgets.chart.bar.bar-width
+ + +
%
+ +
-
- {{ 'widgets.time-series-chart.series.bar.label' | translate }} +
+ {{ 'widgets.chart.bar.label' | translate }}
- - {{ seriesLabelPositionTranslations.get(position) | translate }} + + {{ chartLabelPositionTranslations.get(position) | translate }} @@ -61,16 +68,16 @@
- {{ 'widgets.time-series-chart.series.bar.label-background' | translate }} + {{ 'widgets.chart.bar.label-background' | translate }}
- - + title="widgets.chart.background" + fillNoneTitle="widgets.chart.fill-type-solid"> + diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.ts similarity index 67% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.ts rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.ts index a2f8fd01c3..6fef380a40 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-bar-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { Component, forwardRef, Input, OnInit, Optional } from '@angular/core'; import { ControlValueAccessor, NG_VALUE_ACCESSOR, @@ -22,52 +22,72 @@ import { UntypedFormGroup, Validators } from '@angular/forms'; -import { - BarSeriesSettings, - seriesLabelPositions, - seriesLabelPositionTranslations -} from '@home/components/widget/lib/chart/time-series-chart.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { merge } from 'rxjs'; import { formatValue, isDefinedAndNotNull } from '@core/utils'; import { DataKeyConfigComponent } from '@home/components/widget/config/data-key-config.component'; +import { + ChartBarSettings, + ChartLabelPosition, + chartLabelPositions, + chartLabelPositionTranslations, + PieChartLabelPosition, + pieChartLabelPositions, + pieChartLabelPositionTranslations +} from '@home/components/widget/lib/chart/chart.models'; +import { coerceBoolean } from '@shared/decorators/coercion'; @Component({ - selector: 'tb-time-series-chart-bar-settings', - templateUrl: './time-series-chart-bar-settings.component.html', - styleUrls: ['./../widget-settings.scss'], + selector: 'tb-chart-bar-settings', + templateUrl: './chart-bar-settings.component.html', + styleUrls: ['./../../widget-settings.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => TimeSeriesChartBarSettingsComponent), + useExisting: forwardRef(() => ChartBarSettingsComponent), multi: true } ] }) -export class TimeSeriesChartBarSettingsComponent implements OnInit, ControlValueAccessor { +export class ChartBarSettingsComponent implements OnInit, ControlValueAccessor { - seriesLabelPositions = seriesLabelPositions; + chartLabelPositions: (ChartLabelPosition | PieChartLabelPosition)[]; - seriesLabelPositionTranslations = seriesLabelPositionTranslations; + chartLabelPositionTranslations: Map; labelPreviewFn = this._labelPreviewFn.bind(this); @Input() disabled: boolean; - private modelValue: BarSeriesSettings; + @Input() + @coerceBoolean() + series = true; + + @Input() + @coerceBoolean() + pieLabelPosition = false; + + private modelValue: ChartBarSettings; private propagateChange = null; public barSettingsFormGroup: UntypedFormGroup; constructor(protected store: Store, - private dataKeyConfigComponent: DataKeyConfigComponent, + @Optional() private dataKeyConfigComponent: DataKeyConfigComponent, private fb: UntypedFormBuilder) { } ngOnInit(): void { + if (this.pieLabelPosition) { + this.chartLabelPositions = pieChartLabelPositions; + this.chartLabelPositionTranslations = pieChartLabelPositionTranslations; + } else { + this.chartLabelPositions = chartLabelPositions; + this.chartLabelPositionTranslations = chartLabelPositionTranslations; + } this.barSettingsFormGroup = this.fb.group({ showBorder: [null, []], borderWidth: [null, [Validators.min(0)]], @@ -80,6 +100,10 @@ export class TimeSeriesChartBarSettingsComponent implements OnInit, ControlValue labelBackground: [null, []], backgroundSettings: [null, []] }); + if (!this.series) { + this.barSettingsFormGroup.addControl('barWidth', this.fb.control(null, + [Validators.min(0), Validators.max(100)])); + } this.barSettingsFormGroup.valueChanges.subscribe(() => { this.updateModel(); }); @@ -108,7 +132,7 @@ export class TimeSeriesChartBarSettingsComponent implements OnInit, ControlValue } } - writeValue(value: BarSeriesSettings): void { + writeValue(value: ChartBarSettings): void { this.modelValue = value; this.barSettingsFormGroup.patchValue( value, {emitEvent: false} @@ -150,11 +174,15 @@ export class TimeSeriesChartBarSettingsComponent implements OnInit, ControlValue } private _labelPreviewFn(): string { - const dataKey = this.dataKeyConfigComponent.modelValue; - const widgetConfig = this.dataKeyConfigComponent.widgetConfig; - const units = dataKey.units && dataKey.units.length ? dataKey.units : widgetConfig.config.units; - const decimals = isDefinedAndNotNull(dataKey.decimals) ? dataKey.decimals : - (isDefinedAndNotNull(widgetConfig.config.decimals) ? widgetConfig.config.decimals : 2); - return formatValue(22, decimals, units, false); + if (this.series) { + const dataKey = this.dataKeyConfigComponent.modelValue; + const widgetConfig = this.dataKeyConfigComponent.widgetConfig; + const units = dataKey.units && dataKey.units.length ? dataKey.units : widgetConfig.config.units; + const decimals = isDefinedAndNotNull(dataKey.decimals) ? dataKey.decimals : + (isDefinedAndNotNull(widgetConfig.config.decimals) ? widgetConfig.config.decimals : 2); + return formatValue(22, decimals, units, false); + } else { + return 'Wind'; + } } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.html similarity index 78% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.html rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.html index 99e3c1be8f..d79e84d98e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.html @@ -20,28 +20,28 @@
{{ title | translate }}
- {{ seriesFillTypeTranslationMap.get(type) | translate }} + {{ chartFillTypeTranslationMap.get(type) | translate }}
- +
-
widgets.time-series-chart.series.opacity
+
widgets.chart.opacity
 - +
-
widgets.time-series-chart.series.gradient-stops
+
widgets.chart.gradient-stops
-
widgets.time-series-chart.series.gradient-start
+
widgets.chart.gradient-start
-
widgets.time-series-chart.series.gradient-end
+
widgets.chart.gradient-end
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.ts similarity index 70% rename from ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.ts rename to ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.ts index dd7f4dd7f6..718c4cce02 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/chart-fill-settings.component.ts @@ -22,45 +22,45 @@ import { UntypedFormGroup, Validators } from '@angular/forms'; -import { - SeriesFillSettings, - SeriesFillType, - seriesFillTypes, - seriesFillTypeTranslations -} from '@home/components/widget/lib/chart/time-series-chart.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; +import { + ChartFillSettings, + ChartFillType, + chartFillTypes, + chartFillTypeTranslations +} from '@home/components/widget/lib/chart/chart.models'; @Component({ - selector: 'tb-time-series-chart-fill-settings', - templateUrl: './time-series-chart-fill-settings.component.html', + selector: 'tb-chart-fill-settings', + templateUrl: './chart-fill-settings.component.html', styleUrls: ['./../../widget-settings.scss'], providers: [ { provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => TimeSeriesChartFillSettingsComponent), + useExisting: forwardRef(() => ChartFillSettingsComponent), multi: true } ] }) -export class TimeSeriesChartFillSettingsComponent implements OnInit, ControlValueAccessor { +export class ChartFillSettingsComponent implements OnInit, ControlValueAccessor { - seriesFillTypes = seriesFillTypes; + chartFillTypes = chartFillTypes; - seriesFillTypeTranslationMap: Map = new Map([]); + chartFillTypeTranslationMap: Map = new Map([]); - SeriesFillType = SeriesFillType; + ChartFillType = ChartFillType; @Input() disabled: boolean; @Input() - title = 'widgets.time-series-chart.series.fill'; + title = 'widgets.chart.fill'; @Input() - fillNoneTitle = 'widgets.time-series-chart.series.fill-type-none'; + fillNoneTitle = 'widgets.chart.fill-type-none'; - private modelValue: SeriesFillSettings; + private modelValue: ChartFillSettings; private propagateChange = null; @@ -85,14 +85,14 @@ export class TimeSeriesChartFillSettingsComponent implements OnInit, ControlValu this.fillSettingsFormGroup.get('type').valueChanges.subscribe(() => { this.updateValidators(); }); - for (const type of seriesFillTypes) { + for (const type of chartFillTypes) { let translation: string; - if (type === SeriesFillType.none) { + if (type === ChartFillType.none) { translation = this.fillNoneTitle; } else { - translation = seriesFillTypeTranslations.get(type); + translation = chartFillTypeTranslations.get(type); } - this.seriesFillTypeTranslationMap.set(type, translation); + this.chartFillTypeTranslationMap.set(type, translation); } } @@ -113,7 +113,7 @@ export class TimeSeriesChartFillSettingsComponent implements OnInit, ControlValu } } - writeValue(value: SeriesFillSettings): void { + writeValue(value: ChartFillSettings): void { this.modelValue = value; this.fillSettingsFormGroup.patchValue( value, {emitEvent: false} @@ -122,21 +122,21 @@ export class TimeSeriesChartFillSettingsComponent implements OnInit, ControlValu } private updateValidators() { - const type: SeriesFillType = this.fillSettingsFormGroup.get('type').value; - if (type === SeriesFillType.none) { + const type: ChartFillType = this.fillSettingsFormGroup.get('type').value; + if (type === ChartFillType.none) { this.fillSettingsFormGroup.get('opacity').disable({emitEvent: false}); this.fillSettingsFormGroup.get('gradient').disable({emitEvent: false}); - } else if (type === SeriesFillType.opacity) { + } else if (type === ChartFillType.opacity) { this.fillSettingsFormGroup.get('opacity').enable({emitEvent: false}); this.fillSettingsFormGroup.get('gradient').disable({emitEvent: false}); - } else if (type === SeriesFillType.gradient) { + } else if (type === ChartFillType.gradient) { this.fillSettingsFormGroup.get('opacity').disable({emitEvent: false}); this.fillSettingsFormGroup.get('gradient').enable({emitEvent: false}); } } private updateModel() { - const value: SeriesFillSettings = this.fillSettingsFormGroup.getRawValue(); + const value: ChartFillSettings = this.fillSettingsFormGroup.getRawValue(); this.modelValue = value; this.propagateChange(this.modelValue); } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.html index cd240ba8e4..7ee338bf21 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings.component.html @@ -145,7 +145,7 @@
+ type="number" min="0" placeholder="{{ 'widgets.chart.chart-axis.scale-auto' | translate }}">
@@ -154,24 +154,24 @@
+ type="number" min="1" placeholder="{{ 'widgets.chart.chart-axis.scale-auto' | translate }}">
-
widgets.time-series-chart.axis.scale
+
widgets.chart.chart-axis.scale
-
widgets.time-series-chart.axis.scale-min
+
widgets.chart.chart-axis.scale-min
+ type="number" placeholder="{{ 'widgets.chart.chart-axis.scale-auto' | translate }}"> -
widgets.time-series-chart.axis.scale-max
+
widgets.chart.chart-axis.scale-max
+ type="number" placeholder="{{ 'widgets.chart.chart-axis.scale-auto' | translate }}">
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.html index 4017a0ca2b..293952a342 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.html @@ -103,8 +103,8 @@
- - {{ echartsShapeTranslations.get(shape) | translate }} + + {{ chartShapeTranslations.get(shape) | translate }} @@ -120,8 +120,8 @@
- - {{ echartsShapeTranslations.get(shape) | translate }} + + {{ chartShapeTranslations.get(shape) | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.ts index 766b8b7ccf..e868e1f2e9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings-panel.component.ts @@ -25,15 +25,11 @@ import { timeSeriesThresholdLabelPositions, timeSeriesThresholdLabelPositionTranslations } from '@home/components/widget/lib/chart/time-series-chart.models'; -import { - EChartsShape, - echartsShapes, - echartsShapeTranslations -} from '@home/components/widget/lib/chart/echarts-widget.models'; import { merge } from 'rxjs'; import { WidgetConfig } from '@shared/models/widget.models'; import { formatValue, isDefinedAndNotNull } from '@core/utils'; import { coerceBoolean } from '@shared/decorators/coercion'; +import { ChartShape, chartShapes, chartShapeTranslations } from '@home/components/widget/lib/chart/chart.models'; @Component({ selector: 'tb-time-series-chart-threshold-settings-panel', @@ -48,9 +44,9 @@ export class TimeSeriesChartThresholdSettingsPanelComponent implements OnInit { timeSeriesLineTypeTranslations = timeSeriesLineTypeTranslations; - echartsShapes = echartsShapes; + chartShapes = chartShapes; - echartsShapeTranslations = echartsShapeTranslations; + chartShapeTranslations = chartShapeTranslations; timeSeriesThresholdLabelPositions = timeSeriesThresholdLabelPositions; @@ -127,8 +123,8 @@ export class TimeSeriesChartThresholdSettingsPanelComponent implements OnInit { private updateValidators() { const showLabel: boolean = this.thresholdSettingsFormGroup.get('showLabel').value; const enableLabelBackground: boolean = this.thresholdSettingsFormGroup.get('enableLabelBackground').value; - const startSymbol: EChartsShape = this.thresholdSettingsFormGroup.get('startSymbol').value; - const endSymbol: EChartsShape = this.thresholdSettingsFormGroup.get('endSymbol').value; + const startSymbol: ChartShape = this.thresholdSettingsFormGroup.get('startSymbol').value; + const endSymbol: ChartShape = this.thresholdSettingsFormGroup.get('endSymbol').value; if (showLabel) { this.thresholdSettingsFormGroup.get('labelPosition').enable({emitEvent: false}); this.thresholdSettingsFormGroup.get('labelFont').enable({emitEvent: false}); @@ -146,12 +142,12 @@ export class TimeSeriesChartThresholdSettingsPanelComponent implements OnInit { this.thresholdSettingsFormGroup.get('enableLabelBackground').disable({emitEvent: false}); this.thresholdSettingsFormGroup.get('labelBackground').disable({emitEvent: false}); } - if (startSymbol === EChartsShape.none) { + if (startSymbol === ChartShape.none) { this.thresholdSettingsFormGroup.get('startSymbolSize').disable({emitEvent: false}); } else { this.thresholdSettingsFormGroup.get('startSymbolSize').enable({emitEvent: false}); } - if (endSymbol === EChartsShape.none) { + if (endSymbol === ChartShape.none) { this.thresholdSettingsFormGroup.get('endSymbolSize').disable({emitEvent: false}); } else { this.thresholdSettingsFormGroup.get('endSymbolSize').enable({emitEvent: false}); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.html index 0b1907033f..7291a2d4f7 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/chart/time-series-chart-y-axis-row.component.html @@ -32,12 +32,12 @@
- +
- +
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts index 31c8f60c12..bbe3dfb20c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts @@ -119,8 +119,8 @@ import { TimeSeriesChartAxisSettingsPanelComponent } from '@home/components/widget/lib/settings/common/chart/time-series-chart-axis-settings-panel.component'; import { - EchartsAnimationSettingsComponent -} from '@home/components/widget/lib/settings/common/chart/echarts-animation-settings.component'; + ChartAnimationSettingsComponent +} from '@home/components/widget/lib/settings/common/chart/chart-animation-settings.component'; import { AutoDateFormatSettingsPanelComponent } from '@home/components/widget/lib/settings/common/auto-date-format-settings-panel.component'; @@ -128,8 +128,8 @@ import { AutoDateFormatSettingsComponent } from '@home/components/widget/lib/settings/common/auto-date-format-settings.component'; import { - TimeSeriesChartFillSettingsComponent -} from '@home/components/widget/lib/settings/common/chart/time-series-chart-fill-settings.component'; + ChartFillSettingsComponent +} from '@home/components/widget/lib/settings/common/chart/chart-fill-settings.component'; import { TimeSeriesChartThresholdSettingsComponent } from '@home/components/widget/lib/settings/common/chart/time-series-chart-threshold-settings.component'; @@ -148,6 +148,7 @@ import { import { StatusWidgetStateSettingsComponent } from '@home/components/widget/lib/settings/common/indicator/status-widget-state-settings.component'; +import { ChartBarSettingsComponent } from '@home/components/widget/lib/settings/common/chart/chart-bar-settings.component'; @NgModule({ declarations: [ @@ -195,8 +196,9 @@ import { TimeSeriesChartYAxisRowComponent, TimeSeriesChartAxisSettingsPanelComponent, TimeSeriesChartAxisSettingsButtonComponent, - EchartsAnimationSettingsComponent, - TimeSeriesChartFillSettingsComponent, + ChartAnimationSettingsComponent, + ChartFillSettingsComponent, + ChartBarSettingsComponent, TimeSeriesChartThresholdSettingsComponent, TimeSeriesChartStatesPanelComponent, TimeSeriesChartStateRowComponent, @@ -255,8 +257,9 @@ import { TimeSeriesChartYAxisRowComponent, TimeSeriesChartAxisSettingsPanelComponent, TimeSeriesChartAxisSettingsButtonComponent, - EchartsAnimationSettingsComponent, - TimeSeriesChartFillSettingsComponent, + ChartAnimationSettingsComponent, + ChartFillSettingsComponent, + ChartBarSettingsComponent, TimeSeriesChartThresholdSettingsComponent, TimeSeriesChartStatesPanelComponent, TimeSeriesChartStateRowComponent, diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts index fa34b2c358..db0973a5bc 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/widget-settings.module.ts @@ -336,9 +336,6 @@ import { import { TimeSeriesChartLineSettingsComponent } from '@home/components/widget/lib/settings/chart/time-series-chart-line-settings.component'; -import { - TimeSeriesChartBarSettingsComponent -} from '@home/components/widget/lib/settings/chart/time-series-chart-bar-settings.component'; import { TimeSeriesChartWidgetSettingsComponent } from '@home/components/widget/lib/settings/chart/time-series-chart-widget-settings.component'; @@ -348,6 +345,12 @@ import { import { PieChartWidgetSettingsComponent } from '@home/components/widget/lib/settings/chart/pie-chart-widget-settings.component'; +import { + BarChartWidgetSettingsComponent +} from '@home/components/widget/lib/settings/chart/bar-chart-widget-settings.component'; +import { + PolarAreaChartWidgetSettingsComponent +} from '@home/components/widget/lib/settings/chart/polar-area-chart-widget-settings.component'; @NgModule({ declarations: [ @@ -469,10 +472,11 @@ import { ToggleButtonWidgetSettingsComponent, TimeSeriesChartKeySettingsComponent, TimeSeriesChartLineSettingsComponent, - TimeSeriesChartBarSettingsComponent, TimeSeriesChartWidgetSettingsComponent, StatusWidgetSettingsComponent, - PieChartWidgetSettingsComponent + PieChartWidgetSettingsComponent, + BarChartWidgetSettingsComponent, + PolarAreaChartWidgetSettingsComponent ], imports: [ CommonModule, @@ -599,10 +603,11 @@ import { ToggleButtonWidgetSettingsComponent, TimeSeriesChartKeySettingsComponent, TimeSeriesChartLineSettingsComponent, - TimeSeriesChartBarSettingsComponent, TimeSeriesChartWidgetSettingsComponent, StatusWidgetSettingsComponent, - PieChartWidgetSettingsComponent + PieChartWidgetSettingsComponent, + BarChartWidgetSettingsComponent, + PolarAreaChartWidgetSettingsComponent ] }) export class WidgetSettingsModule { @@ -697,5 +702,7 @@ export const widgetSettingsComponentsMap: {[key: string]: Type Date: Thu, 25 Apr 2024 20:13:20 +0200 Subject: [PATCH 61/72] SecurityConfiguration: deprecated authorizeRequests replaced with authorizeHttpRequests --- .../server/config/TbRuleEngineSecurityConfiguration.java | 7 ++++--- .../server/config/ThingsboardSecurityConfiguration.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java index e1ab755036..15d33c7fcb 100644 --- a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java @@ -40,9 +40,10 @@ public class TbRuleEngineSecurityConfiguration { .frameOptions(config -> {}).disable()) .cors(cors -> {}) .csrf(AbstractHttpConfigurer::disable) - .authorizeRequests() - .requestMatchers("/actuator/prometheus").permitAll() - .anyRequest().authenticated(); + .authorizeHttpRequests(config -> config + .requestMatchers("/actuator/prometheus").permitAll()) + .authorizeHttpRequests(config -> config + .anyRequest().authenticated()); return http.build(); } } diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index c27ca35313..67c14ce656 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -209,7 +209,7 @@ public class ThingsboardSecurityConfiguration { .csrf(AbstractHttpConfigurer::disable) .exceptionHandling(config -> {}) .sessionManagement(config -> config.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) - .authorizeRequests(config -> config + .authorizeHttpRequests(config -> config .requestMatchers(WEBJARS_ENTRY_POINT).permitAll() // Webjars .requestMatchers(DEVICE_API_ENTRY_POINT).permitAll() // Device HTTP Transport API .requestMatchers(FORM_BASED_LOGIN_ENTRY_POINT).permitAll() // Login end-point @@ -218,7 +218,7 @@ public class ThingsboardSecurityConfiguration { .requestMatchers(MAIL_OAUTH2_PROCESSING_ENTRY_POINT).permitAll() // Mail oauth2 code processing url .requestMatchers(DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT).permitAll() // Device connectivity certificate (public) .requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll()) // static resources, user activation and password reset end-points - .authorizeRequests(config -> config + .authorizeHttpRequests(config -> config .requestMatchers(WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points .requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated()) // Protected API End-points .exceptionHandling(config -> config.accessDeniedHandler(restAccessDeniedHandler)) From 228bec0a9c27a443fd89b5be8847dd82fc8522a1 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 26 Apr 2024 10:49:38 +0300 Subject: [PATCH 62/72] UI: Change locale code ar_AR to ar_AE --- .../{locale.constant-ar_AR.json => locale.constant-ar_AE.json} | 0 ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ui-ngx/src/assets/locale/{locale.constant-ar_AR.json => locale.constant-ar_AE.json} (100%) diff --git a/ui-ngx/src/assets/locale/locale.constant-ar_AR.json b/ui-ngx/src/assets/locale/locale.constant-ar_AE.json similarity index 100% rename from ui-ngx/src/assets/locale/locale.constant-ar_AR.json rename to ui-ngx/src/assets/locale/locale.constant-ar_AE.json diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 7485ba64f1..7de9b02687 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -7291,7 +7291,7 @@ "language": { "language": "Language", "locales": { - "ar_AR": "اَلْعَرَبِيَّةُ", + "ar_AE": "اَلْعَرَبِيَّةُ", "ca_ES": "Catalan", "cs_CZ": "Česky", "da_DK": "Dansk", From 1ae1e90f868e6bd67750a4d354957ab80794ef42 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 26 Apr 2024 13:05:06 +0300 Subject: [PATCH 63/72] Minor refactoring --- .../server/service/housekeeper/HousekeeperService.java | 1 - .../thingsboard/server/controller/AbstractWebTest.java | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java index 898ae74d3f..06e38d0160 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -117,7 +117,6 @@ public class HousekeeperService { throw new IllegalArgumentException("Unsupported task type " + taskType); } - try { long startTs = System.currentTimeMillis(); Future future = taskExecutor.submit(() -> { diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 706d3a3e5e..96e52d6c12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -69,7 +69,6 @@ import org.thingsboard.server.actors.device.DeviceActorMessageProcessor; import org.thingsboard.server.actors.device.SessionInfo; import org.thingsboard.server.actors.device.ToDeviceRpcRequestMetadata; import org.thingsboard.server.actors.service.DefaultActorService; -import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -102,8 +101,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; @@ -441,11 +438,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } protected Tenant saveTenant(Tenant tenant) throws Exception { - tenant = doPost("/api/tenant", tenant, Tenant.class); - if (tenant.getId() == null) { - attributesService.save(TenantId.SYS_TENANT_ID, tenant.getId(), AttributeScope.SERVER_SCOPE, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry("test", "test"))).get(); // creating marker attr to later know when Housekeeper finishes tenant cleanup - } - return tenant; + return doPost("/api/tenant", tenant, Tenant.class); } protected void loginDifferentCustomer() throws Exception { From 872309e6a2ea5f6760cb350609d0b487a9683789 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 26 Apr 2024 16:02:49 +0200 Subject: [PATCH 64/72] added clear cache for upgrade from 3.6.4 to 3.7.0 --- .../service/install/update/DefaultCacheCleanupService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index 42e1cb2cbf..5a1d2fe38d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -55,7 +55,11 @@ public class DefaultCacheCleanupService implements CacheCleanupService { clearCacheByName(RESOURCE_INFO_CACHE); break; case "3.6.3": - log.info("Clearing cache to upgrade from version 3.6.3 to 3.7.0"); + log.info("Clearing cache to upgrade from version 3.6.3 to 3.6.4"); + clearAll(); + break; + case "3.6.4": + log.info("Clearing cache to upgrade from version 3.6.4 to 3.7.0"); clearAll(); break; default: From 94b46958f9f7d92d5f9024fd5768b5723f261939 Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Fri, 26 Apr 2024 17:29:43 +0300 Subject: [PATCH 65/72] Improvement for EntityStateSourcingListener --- .../server/service/entitiy/EntityStateSourcingListener.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index c013650fa7..6027ad92d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -73,6 +73,9 @@ public class EntityStateSourcingListener { public void handleEvent(SaveEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); + if (entityId == null) { + return; + } EntityType entityType = entityId.getEntityType(); log.debug("[{}][{}][{}] Handling entity save event: {}", tenantId, entityType, entityId, event); boolean isCreated = event.getCreated() != null && event.getCreated(); From 4e6ab036de4d79cb4037d8a485035971b188415c Mon Sep 17 00:00:00 2001 From: nick Date: Fri, 26 Apr 2024 18:09:10 +0300 Subject: [PATCH 66/72] fix bug: coap dtls.connection_id_length --- .../org/thingsboard/server/coapserver/TbCoapDtlsSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java index f27131b7ab..9c6e606af5 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java @@ -61,7 +61,7 @@ public class TbCoapDtlsSettings { @Value("${transport.coap.dtls.retransmission_timeout:9000}") private int dtlsRetransmissionTimeout; - @Value("${transport.coap.dtls.connection_id_length}") + @Value("${transport.coap.dtls.connection_id_length:}") private Integer cIdLength; @Bean From 115596da28f9f196b1f8cb19486d6bb274f3cd9f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 26 Apr 2024 17:19:24 +0200 Subject: [PATCH 67/72] coap.dtls.connection_id_length added default null value to tolerate yaml inconsistency case --- .../org/thingsboard/server/coapserver/TbCoapDtlsSettings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java index f27131b7ab..9c6e606af5 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java @@ -61,7 +61,7 @@ public class TbCoapDtlsSettings { @Value("${transport.coap.dtls.retransmission_timeout:9000}") private int dtlsRetransmissionTimeout; - @Value("${transport.coap.dtls.connection_id_length}") + @Value("${transport.coap.dtls.connection_id_length:}") private Integer cIdLength; @Bean From 8686a92944dcea231812644201983820c6ca985e Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Fri, 26 Apr 2024 20:18:53 +0300 Subject: [PATCH 68/72] Replace static mocks with consumers in DefaultRuleEngineDeviceStateManagerTest --- ...faultRuleEngineDeviceStateManagerTest.java | 94 ++++++++++--------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java b/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java index c97305c170..1fae187a15 100644 --- a/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java +++ b/application/src/test/java/org/thingsboard/server/service/state/DefaultRuleEngineDeviceStateManagerTest.java @@ -40,6 +40,8 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import java.util.Optional; import java.util.UUID; +import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.stream.Stream; import static org.mockito.ArgumentMatchers.any; @@ -53,13 +55,13 @@ import static org.mockito.Mockito.never; public class DefaultRuleEngineDeviceStateManagerTest { @Mock - private static DeviceStateService deviceStateServiceMock; + private DeviceStateService deviceStateServiceMock; @Mock - private static TbCallback tbCallbackMock; + private TbCallback tbCallbackMock; @Mock - private static TbClusterService clusterServiceMock; + private TbClusterService clusterServiceMock; @Mock - private static TbQueueMsgMetadata metadataMock; + private TbQueueMsgMetadata metadataMock; @Mock private TbServiceInfoProvider serviceInfoProviderMock; @@ -67,9 +69,9 @@ public class DefaultRuleEngineDeviceStateManagerTest { private PartitionService partitionServiceMock; @Captor - private static ArgumentCaptor queueCallbackCaptor; + private ArgumentCaptor queueCallbackCaptor; - private static DefaultRuleEngineDeviceStateManager deviceStateManager; + private DefaultRuleEngineDeviceStateManager deviceStateManager; private static final TenantId TENANT_ID = TenantId.fromUUID(UUID.fromString("57ab2e6c-bc4c-11ee-a506-0242ac120002")); private static final DeviceId DEVICE_ID = DeviceId.fromString("74a9053e-bc4c-11ee-a506-0242ac120002"); @@ -87,15 +89,17 @@ public class DefaultRuleEngineDeviceStateManagerTest { @DisplayName("Given event should be routed to local service and event processed has succeeded, " + "when onDeviceX() is called, then should route event to local service and call onSuccess() callback.") @MethodSource - public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback(Runnable onDeviceAction, Runnable actionVerification) { + public void givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback( + BiConsumer onDeviceAction, Consumer actionVerification + ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); given(partitionServiceMock.resolve(ServiceType.TB_CORE, TENANT_ID, DEVICE_ID)).willReturn(MY_TPI); - onDeviceAction.run(); + onDeviceAction.accept(deviceStateManager, tbCallbackMock); // THEN - actionVerification.run(); + actionVerification.accept(deviceStateServiceMock); then(clusterServiceMock).shouldHaveNoInteractions(); then(tbCallbackMock).should().onSuccess(); @@ -105,20 +109,20 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToLocalAndProcessingSuccess_whenOnDeviceAction_thenShouldCallLocalServiceAndSuccessCallback() { return Stream.of( Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -128,19 +132,19 @@ public class DefaultRuleEngineDeviceStateManagerTest { "when onDeviceX() is called, then should route event to local service and call onFailure() callback.") @MethodSource public void givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback( - Runnable exceptionThrowSetup, Runnable onDeviceAction, Runnable actionVerification + Consumer exceptionThrowSetup, BiConsumer onDeviceAction, Consumer actionVerification ) { // GIVEN given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(true); given(partitionServiceMock.resolve(ServiceType.TB_CORE, TENANT_ID, DEVICE_ID)).willReturn(MY_TPI); - exceptionThrowSetup.run(); + exceptionThrowSetup.accept(deviceStateServiceMock); // WHEN - onDeviceAction.run(); + onDeviceAction.accept(deviceStateManager, tbCallbackMock); // THEN - actionVerification.run(); + actionVerification.accept(deviceStateServiceMock); then(clusterServiceMock).shouldHaveNoInteractions(); then(tbCallbackMock).should(never()).onSuccess(); @@ -150,24 +154,24 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToLocalAndProcessingFailure_whenOnDeviceAction_thenShouldCallLocalServiceAndFailureCallback() { return Stream.of( Arguments.of( - (Runnable) () -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (Runnable) () -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (Runnable) () -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS), - (Runnable) () -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS) ), Arguments.of( - (Runnable) () -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS), - (Runnable) () -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) + (Consumer) deviceStateServiceMock -> doThrow(RUNTIME_EXCEPTION).when(deviceStateServiceMock).onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS), + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (Consumer) deviceStateServiceMock -> then(deviceStateServiceMock).should().onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS) ) ); } @@ -176,16 +180,18 @@ public class DefaultRuleEngineDeviceStateManagerTest { @DisplayName("Given event should be routed to external service, " + "when onDeviceX() is called, then should send correct queue message to external service with correct callback object.") @MethodSource - public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback(Runnable onDeviceAction, Runnable actionVerification) { + public void givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback( + BiConsumer onDeviceAction, BiConsumer> actionVerification + ) { // WHEN ReflectionTestUtils.setField(deviceStateManager, "deviceStateService", Optional.empty()); given(serviceInfoProviderMock.isService(ServiceType.TB_CORE)).willReturn(false); given(partitionServiceMock.resolve(ServiceType.TB_CORE, TENANT_ID, DEVICE_ID)).willReturn(EXTERNAL_TPI); - onDeviceAction.run(); + onDeviceAction.accept(deviceStateManager, tbCallbackMock); // THEN - actionVerification.run(); + actionVerification.accept(clusterServiceMock, queueCallbackCaptor); TbQueueCallback callback = queueCallbackCaptor.getValue(); callback.onSuccess(metadataMock); @@ -197,8 +203,8 @@ public class DefaultRuleEngineDeviceStateManagerTest { private static Stream givenRoutedToExternal_whenOnDeviceAction_thenShouldSendQueueMsgToExternalServiceWithCorrectCallback() { return Stream.of( Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> { + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceConnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceConnectMsg = TransportProtos.DeviceConnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) @@ -213,8 +219,8 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> { + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceActivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) @@ -229,8 +235,8 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> { + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceDisconnect(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceDisconnectMsg = TransportProtos.DeviceDisconnectProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) @@ -245,8 +251,8 @@ public class DefaultRuleEngineDeviceStateManagerTest { } ), Arguments.of( - (Runnable) () -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), - (Runnable) () -> { + (BiConsumer) (deviceStateManager, tbCallbackMock) -> deviceStateManager.onDeviceInactivity(TENANT_ID, DEVICE_ID, EVENT_TS, tbCallbackMock), + (BiConsumer>) (clusterServiceMock, queueCallbackCaptor) -> { var deviceInactivityMsg = TransportProtos.DeviceInactivityProto.newBuilder() .setTenantIdMSB(TENANT_ID.getId().getMostSignificantBits()) .setTenantIdLSB(TENANT_ID.getId().getLeastSignificantBits()) From abf85cb9b859c649c2f51e23b4c49b194a51aedb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 26 Apr 2024 22:03:26 +0200 Subject: [PATCH 69/72] SecurityConfiguration fixed and refactored based on manual testing results on QA env --- .../server/config/TbRuleEngineSecurityConfiguration.java | 3 +-- .../server/config/ThingsboardSecurityConfiguration.java | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java index 15d33c7fcb..15171ba0c7 100644 --- a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java @@ -41,8 +41,7 @@ public class TbRuleEngineSecurityConfiguration { .cors(cors -> {}) .csrf(AbstractHttpConfigurer::disable) .authorizeHttpRequests(config -> config - .requestMatchers("/actuator/prometheus").permitAll()) - .authorizeHttpRequests(config -> config + .requestMatchers("/actuator/prometheus").permitAll() .anyRequest().authenticated()); return http.build(); } diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 67c14ce656..eda8d3e89d 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -217,10 +217,11 @@ public class ThingsboardSecurityConfiguration { .requestMatchers(TOKEN_REFRESH_ENTRY_POINT).permitAll() // Token refresh end-point .requestMatchers(MAIL_OAUTH2_PROCESSING_ENTRY_POINT).permitAll() // Mail oauth2 code processing url .requestMatchers(DEVICE_CONNECTIVITY_CERTIFICATE_DOWNLOAD_ENTRY_POINT).permitAll() // Device connectivity certificate (public) - .requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll()) // static resources, user activation and password reset end-points - .authorizeHttpRequests(config -> config + .requestMatchers(NON_TOKEN_BASED_AUTH_ENTRY_POINTS).permitAll() // static resources, user activation and password reset end-points .requestMatchers(WS_ENTRY_POINT).permitAll() // Protected WebSocket API End-points .requestMatchers(TOKEN_BASED_AUTH_ENTRY_POINT).authenticated()) // Protected API End-points + .formLogin(form -> form + .loginPage("/login").permitAll()) .exceptionHandling(config -> config.accessDeniedHandler(restAccessDeniedHandler)) .addFilterBefore(buildRestLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildRestPublicLoginProcessingFilter(), UsernamePasswordAuthenticationFilter.class) @@ -243,7 +244,7 @@ public class ThingsboardSecurityConfiguration { @Bean @ConditionalOnMissingBean(CorsFilter.class) public CorsFilter corsFilter(@Autowired MvcCorsProperties mvcCorsProperties) { - if (mvcCorsProperties.getMappings().size() == 0) { + if (mvcCorsProperties.getMappings().isEmpty()) { return new CorsFilter(new UrlBasedCorsConfigurationSource()); } else { UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); From 20de60256cdc62960a4dc484b14a1a44f724afdc Mon Sep 17 00:00:00 2001 From: Andrii Landiak Date: Mon, 29 Apr 2024 10:38:21 +0300 Subject: [PATCH 70/72] Add fail-fast for DeleteEntityEvent on EntityStateSourcingListener --- .../server/service/entitiy/EntityStateSourcingListener.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index 6027ad92d9..4a7c1e1e87 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -125,6 +125,9 @@ public class EntityStateSourcingListener { public void handleEvent(DeleteEntityEvent event) { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); + if (entityId == null) { + return; + } EntityType entityType = entityId.getEntityType(); if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); From 9d159526d919eaeb9a674dee1caf5ef3c0f3c5f2 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 29 Apr 2024 11:26:18 +0300 Subject: [PATCH 71/72] added tests for EntityService --- .../server/dao/service/EntityServiceTest.java | 155 +++++++++++++++++- 1 file changed, 153 insertions(+), 2 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java index d9b94fb26b..1fc2ff2ce6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityServiceTest.java @@ -18,18 +18,28 @@ package org.thingsboard.server.dao.service; import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.hamcrest.Matchers; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.ResultSetExtractor; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.alarm.AlarmApiCallResult; +import org.thingsboard.server.common.data.alarm.AlarmCreateOrUpdateActiveRequest; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; @@ -46,6 +56,7 @@ import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.ApiUsageStateFilter; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; import org.thingsboard.server.common.data.query.AssetTypeFilter; import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; @@ -65,6 +76,7 @@ import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate.StringOperation; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -72,15 +84,20 @@ import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.sql.relation.RelationRepository; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import java.util.ArrayList; @@ -91,12 +108,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; import static org.thingsboard.server.common.data.query.EntityKeyType.ATTRIBUTE; import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIELD; @@ -106,6 +124,7 @@ import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIEL public class EntityServiceTest extends AbstractServiceTest { static final int ENTITY_COUNT = 5; + public static final String TEST_CUSTOMER_NAME = "Test"; @Autowired AssetService assetService; @@ -125,6 +144,27 @@ public class EntityServiceTest extends AbstractServiceTest { RelationService relationService; @Autowired TimeseriesService timeseriesService; + @Autowired + ApiUsageStateService apiUsageStateService; + @Autowired + CustomerService customerService; + @Autowired + DashboardDao dashboardDao; + @Autowired + EntityViewDao entityViewDao; + @Autowired + AlarmService alarmService; + + private CustomerId customerId; + + @Before + public void before() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(TEST_CUSTOMER_NAME); + customer = customerService.saveCustomer(customer); + customerId = customer.getId(); + } @Test public void testCountEntitiesByQuery() throws InterruptedException { @@ -810,7 +850,7 @@ public class EntityServiceTest extends AbstractServiceTest { } void createLoopRelations(TenantId tenantId, String type, EntityId... ids) { - assertThat("ids lenght", ids.length, Matchers.greaterThanOrEqualTo(1)); + assertThat(ids.length).isGreaterThanOrEqualTo(1); //chain all from the head to the tail for (int i = 1; i < ids.length; i++) { relationService.saveRelation(tenantId, new EntityRelation(ids[i - 1], ids[i], type, RelationTypeGroup.COMMON)); @@ -1179,6 +1219,117 @@ public class EntityServiceTest extends AbstractServiceTest { assertEquals(0, result.getTotalElements()); } + @Test + public void testFindEntitiesBySingleEntityFilter() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 3; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(deviceService.saveDevice(device)); + } + + SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); + singleEntityFilter.setSingleEntity(devices.get(0).getId()); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(singleEntityFilter, pageLink, entityFields, null, null); + + PageData result = searchEntities(query); + assertEquals(1, result.getTotalElements()); + + String deviceName = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(deviceName).isEqualTo(devices.get(0).getName()); + } + + @Test + public void testFindEntitiesByApiUsageStateFilter() { + apiUsageStateService.createDefaultApiUsageState(tenantId, customerId); + ApiUsageStateFilter apiUsageStateFilter = new ApiUsageStateFilter(); + apiUsageStateFilter.setCustomerId(customerId); + + List entityFields = List.of( + new EntityKey(EntityKeyType.ENTITY_FIELD, "name") + ); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + EntityDataQuery query = new EntityDataQuery(apiUsageStateFilter, pageLink, entityFields, null, null); + PageData result = searchEntities(query); + assertEquals(1, result.getTotalElements()); + String name = result.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(name).isEqualTo(TEST_CUSTOMER_NAME); + } + + @Test + public void testFindEntitiesByRelationEntityTypeFilter() { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle("Customer Relation Query"); + customer = customerService.saveCustomer(customer); + + final int assetCount = 2; + final int relationsCnt = 4; + final int deviceEntitiesCnt = assetCount * relationsCnt; + + List assets = new ArrayList<>(); + for (int i = 0; i < assetCount; i++) { + Asset building = new Asset(); + building.setTenantId(tenantId); + building.setCustomerId(customer.getId()); + building.setName("Building _" + i); + building.setType("building"); + building = assetService.saveAsset(building); + assets.add(building); + } + + List devices = new ArrayList<>(); + for (int i = 0; i < deviceEntitiesCnt; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customer.getId()); + device.setName("Test device " + i); + device.setType("default"); + Device savedDevice = deviceService.saveDevice(device); + devices.add(savedDevice); + } + + for (int i = 0; i < assetCount; i++) { + for (int j = 0; j < relationsCnt; j++) { + EntityRelation relationEntity = new EntityRelation(); + relationEntity.setFrom(assets.get(i).getId()); + relationEntity.setTo(devices.get(j + (i * relationsCnt)).getId()); + relationEntity.setTypeGroup(RelationTypeGroup.COMMON); + relationEntity.setType("contains"); + relationService.saveRelation(tenantId, relationEntity); + } + } + + RelationEntityTypeFilter relationEntityTypeFilter = new RelationEntityTypeFilter("contains", Collections.singletonList(EntityType.DEVICE)); + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setFilters(Collections.singletonList(relationEntityTypeFilter)); + filter.setDirection(EntitySearchDirection.FROM); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, null); + List keyFiltersEqualString = createStringKeyFilters("name", EntityKeyType.ENTITY_FIELD, StringOperation.STARTS_WITH, "Test device "); + + for (Asset asset : assets) { + filter.setRootEntity(asset.getId()); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, Collections.emptyList(), Collections.emptyList(), keyFiltersEqualString); + PageData relationsResult = entityService.findEntityDataByQuery(tenantId, customer.getId(), query); + long relationsResultCnt = entityService.countEntitiesByQuery(tenantId, customer.getId(), query); + + Assert.assertEquals(relationsCnt, relationsResult.getData().size()); + Assert.assertEquals(relationsCnt, relationsResultCnt); + } + } + @Test public void testFindEntityDataByQuery_filter_device_type_name_ends_with() { List devices = new ArrayList<>(); From 386795337ffb1b0d96ec89dca6b8d042c09163ea Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 29 Apr 2024 11:27:27 +0300 Subject: [PATCH 72/72] Fix partitions removed when tenant is updated --- .../server/common/transport/service/DefaultTransportService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 6280f09cfd..41886bd88a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -1001,7 +1001,6 @@ public class DefaultTransportService extends TransportActivityManager implements break; case TENANT: Tenant tenant = ProtoUtils.fromProto(msg.getTenant()); - partitionService.removeTenant(tenant.getId()); boolean updated = tenantProfileCache.put(tenant.getId(), tenant.getTenantProfileId()); partitionService.evictTenantInfo(tenant.getId()); if (updated) {