diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index 31cb159229..a185b71d56 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -287,7 +287,11 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM state.checkStateSize(ctxId, ctx.getMaxStateSize()); stateSizeChecked = true; if (state.isSizeOk()) { - cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + if (!calculationResult.isEmpty()) { + cfService.pushMsgToRuleEngine(tenantId, entityId, calculationResult, cfIdList, callback); + } else { + callback.onSuccess(); + } if (DebugModeUtil.isDebugAllAvailable(ctx.getCalculatedField())) { systemContext.persistCalculatedFieldDebugEvent(tenantId, ctx.getCfId(), entityId, state.getArguments(), tbMsgId, tbMsgType, JacksonUtil.writeValueAsString(calculationResult.getResult()), null); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java index 8eb27395c1..49acf6917c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java @@ -27,4 +27,11 @@ public final class CalculatedFieldResult { private final AttributeScope scope; private final JsonNode result; + public boolean isEmpty() { + return result == null || result.isMissingNode() || result.isNull() || + (result.isObject() && result.isEmpty()) || + (result.isArray() && result.isEmpty()) || + (result.isTextual() && result.asText().isEmpty()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index c51aaa2e72..0c4352dcea 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -83,7 +83,7 @@ public class CalculatedFieldCtx { for (Map.Entry entry : arguments.entrySet()) { var refId = entry.getValue().getRefEntityId(); var refKey = entry.getValue().getRefEntityKey(); - if (refId == null) { + if (refId == null || refId.equals(calculatedField.getEntityId())) { mainEntityArguments.put(refKey, entry.getKey()); } else { linkedEntityArguments.computeIfAbsent(refId, key -> new HashMap<>()).put(refKey, entry.getKey()); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3cc28ad702..b3022fb9d3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1757,8 +1757,6 @@ queue: stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" - # Statistics printing interval for EDQS - print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java index 9ddf0f0a04..002f668fdc 100644 --- a/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/cf/CalculatedFieldIntegrationTest.java @@ -48,6 +48,9 @@ import static org.awaitility.Awaitility.await; @DaoSqlTest public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTest { + public static final int TIMEOUT = 60; + public static final int POLL_INTERVAL = 1; + @BeforeEach void setUp() throws Exception { loginTenantAdmin(); @@ -86,6 +89,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); await().alias("create CF -> perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -95,6 +99,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -108,6 +113,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); await().alias("update CF output -> perform calculation with updated output").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); @@ -119,6 +125,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); await().alias("update CF argument -> perform calculation with new argument").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); @@ -129,6 +136,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes savedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); await().alias("update CF expression -> perform calculation with new expression").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ArrayNode temperatureF = getServerAttributes(testDevice.getId(), "temperatureF"); assertThat(temperatureF).isNotNull(); @@ -166,6 +174,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); await().alias("create CF -> state is not ready -> no calculation performed").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -175,6 +184,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); await().alias("update telemetry -> perform calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -213,6 +223,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); await().alias("create CF -> perform initial calculation with default value").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -222,6 +233,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); await().alias("update telemetry -> recalculate state").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -277,6 +289,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/calculatedField", calculatedField, CalculatedField.class); await().alias("create CF and perform initial calculation").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -292,6 +305,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":25}")); await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -307,6 +321,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/ASSET/" + asset1.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":15}")); await().alias("update asset 1 telemetry -> recalculate state only for asset 1").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -322,6 +337,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/ASSET/" + asset2.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"y\":5}")); await().alias("update asset 2 telemetry -> recalculate state only for asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 (no changes) ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -339,6 +355,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Asset finalAsset3 = asset3; await().alias("add new entity to profile -> calculate state for new entity").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 3 ArrayNode z3 = getServerAttributes(finalAsset3.getId(), "z"); @@ -349,6 +366,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/attributes/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"x\":20}")); await().alias("update device telemetry -> recalculate state for all assets").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -375,6 +393,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes Asset updatedAsset3 = asset3; await().alias("update device telemetry -> recalculate state for asset 1 and asset 2").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { // result of asset 1 ArrayNode z1 = getServerAttributes(asset1.getId(), "z"); @@ -425,6 +444,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes CalculatedField savedCalculatedField = doPost("/api/calculatedField", calculatedField, CalculatedField.class); await().alias("create CF -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); @@ -434,6 +454,7 @@ public class CalculatedFieldIntegrationTest extends CalculatedFieldControllerTes doPost("/api/plugins/telemetry/DEVICE/" + testDevice.getUuidId() + "/timeseries/" + DataConstants.SERVER_SCOPE, JacksonUtil.toJsonNode("{\"temperature\":30}")); await().alias("update telemetry -> ctx is not initialized -> no calculation perform").atMost(TIMEOUT, TimeUnit.SECONDS) + .pollInterval(POLL_INTERVAL, TimeUnit.SECONDS) .untilAsserted(() -> { ObjectNode fahrenheitTemp = getLatestTelemetry(testDevice.getId(), "fahrenheitTemp"); assertThat(fahrenheitTemp).isNotNull(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 77002cef8e..b0a3518e60 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -28,6 +28,7 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -73,6 +74,7 @@ import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.argThat; @@ -325,6 +327,59 @@ public class BaseQueueControllerTest extends AbstractControllerTest { doDelete("/api/queues/" + queue.getUuidId()).andExpect(status().isOk()); } + @Test + public void testQueueWithReservedName() throws Exception { + loginSysAdmin(); + + // create queue + Queue queue = new Queue(); + queue.setName(DataConstants.CF_QUEUE_NAME); + queue.setTopic("tb_rule_engine.calculated_fields"); + queue.setPollInterval(25); + queue.setPartitions(10); + queue.setTenantId(TenantId.SYS_TENANT_ID); + queue.setConsumerPerPartition(false); + queue.setPackProcessingTimeout(2000); + SubmitStrategy submitStrategy = new SubmitStrategy(); + submitStrategy.setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + queue.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy = new ProcessingStrategy(); + processingStrategy.setType(ProcessingStrategyType.RETRY_ALL); + processingStrategy.setRetries(3); + processingStrategy.setFailurePercentage(0.7); + processingStrategy.setPauseBetweenRetries(3); + processingStrategy.setMaxPauseBetweenRetries(5); + queue.setProcessingStrategy(processingStrategy); + + doPost("/api/queues?serviceType=" + "TB-RULE-ENGINE", queue) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(String.format("The queue name '%s' is not allowed. This name is reserved for internal use. Please choose a different name.", DataConstants.CF_QUEUE_NAME)))); + + // create queue + Queue queue2 = new Queue(); + queue2.setName(DataConstants.CF_STATES_QUEUE_NAME); + queue2.setTopic("tb_rule_engine.calculated_fields"); + queue2.setPollInterval(25); + queue2.setPartitions(10); + queue2.setTenantId(TenantId.SYS_TENANT_ID); + queue2.setConsumerPerPartition(false); + queue2.setPackProcessingTimeout(2000); + SubmitStrategy submitStrategy2 = new SubmitStrategy(); + submitStrategy2.setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR); + queue2.setSubmitStrategy(submitStrategy); + ProcessingStrategy processingStrategy2 = new ProcessingStrategy(); + processingStrategy2.setType(ProcessingStrategyType.RETRY_ALL); + processingStrategy2.setRetries(3); + processingStrategy2.setFailurePercentage(0.7); + processingStrategy2.setPauseBetweenRetries(3); + processingStrategy2.setMaxPauseBetweenRetries(5); + queue2.setProcessingStrategy(processingStrategy); + + doPost("/api/queues?serviceType=" + "TB-RULE-ENGINE", queue2) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(String.format("The queue name '%s' is not allowed. This name is reserved for internal use. Please choose a different name.", DataConstants.CF_STATES_QUEUE_NAME)))); + } + private Queue saveQueue(Queue queue) { return doPost("/api/queues?serviceType=TB_RULE_ENGINE", queue, Queue.class); } diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 4a303e5253..dace159774 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -428,7 +428,6 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(partitionService, "corePartitions", 10); ReflectionTestUtils.setField(partitionService, "cfEventTopic", "tb_cf_event"); ReflectionTestUtils.setField(partitionService, "cfStateTopic", "tb_cf_state"); - ReflectionTestUtils.setField(partitionService, "cfPartitions", 10); ReflectionTestUtils.setField(partitionService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(partitionService, "vcPartitions", 10); ReflectionTestUtils.setField(partitionService, "hashFunctionName", hashFunctionName); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index a36514248c..9ba6c20188 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -289,7 +289,7 @@ public class FieldsUtil { } public static String getText(JsonNode node) { - return node != null ? node.asText() : ""; + return node != null ? node.toString() : ""; } private static UUID getCustomerId(CustomerId customerId) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index 0c55bc50dd..59255b11cc 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -430,6 +430,9 @@ public class TenantRepo { private String getEntityName(EntityId entityId) { EntityType entityType = entityId.getEntityType(); + if (entityType == EntityType.TENANT && entityId.getId().equals(TenantId.NULL_UUID)) { + return ""; + } return switch (entityType) { case CUSTOMER, TENANT -> getEntityMap(entityType).get(entityId.getId()).getFields().getName(); default -> throw new RuntimeException("Unsupported entity type: " + entityType); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java index a12a12dbe3..442453fc93 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java @@ -20,7 +20,6 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.EdqsEventType; @@ -43,18 +42,6 @@ public class EdqsStatsService { private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); private final StatsFactory statsFactory; - @Scheduled(initialDelayString = "${queue.edqs.stats.print-interval-ms:300000}", - fixedDelayString = "${queue.edqs.stats.print-interval-ms:300000}") - private void reportStats() { - if (statsMap.isEmpty()) { - return; - } - String values = statsMap.entrySet().stream() - .map(kv -> "TenantId [" + kv.getKey() + "] stats [" + kv.getValue() + "]") - .collect(Collectors.joining(System.lineSeparator())); - log.info("EDQS Stats: {}", values); - } - public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) .reportEvent(objectType, eventType); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index ba88d446d9..9f1f583108 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -21,6 +21,7 @@ import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; @@ -158,6 +159,9 @@ public abstract class DataValidator> { protected static void validateQueueName(String name) { validateQueueNameOrTopic(name, NAME); + if (DataConstants.CF_QUEUE_NAME.equals(name) || DataConstants.CF_STATES_QUEUE_NAME.equals(name)) { + throw new DataValidationException(String.format("The queue name '%s' is not allowed. This name is reserved for internal use. Please choose a different name.", name)); + } } protected static void validateQueueTopic(String topic) { diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 5767026b11..3862024786 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -143,7 +143,6 @@ function additionalComposeEdqsArgs() { function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log - 799 799 tb-transports/coap/log 799 799 tb-transports/lwm2m/log 799 799 tb-transports/http/log 799 799 tb-transports/mqtt/log @@ -200,29 +199,77 @@ function permissionList() { } function checkFolders() { + CREATE=false + SKIP_CHOWN=false + for i in "$@" + do + case $i in + --create) + CREATE=true + shift + ;; + --skipChown) + SKIP_CHOWN=true + shift + ;; + *) + # unknown option + ;; + esac + done EXIT_CODE=0 PERMISSION_LIST=$(permissionList) || exit $? set -e while read -r USR GRP DIR do - if [ -z "$DIR" ]; then # skip empty lines + IS_EXIST_CHECK_PASSED=false + IS_OWNER_CHECK_PASSED=false + + # skip empty lines + if [ -z "$DIR" ]; then continue fi - MESSAGE="Checking user ${USR} group ${GRP} dir ${DIR}" - if [[ -d "$DIR" ]] && - [[ $(ls -ldn "$DIR" | awk '{print $3}') -eq "$USR" ]] && - [[ $(ls -ldn "$DIR" | awk '{print $4}') -eq "$GRP" ]] - then - MESSAGE="$MESSAGE OK" + + # checks section + echo "Checking if dir ${DIR} exists..." + if [[ -d "$DIR" ]]; then + echo "> OK" + IS_EXIST_CHECK_PASSED=true + if [ "$SKIP_CHOWN" = false ]; then + echo "Checking user ${USR} group ${GRP} ownership for dir ${DIR}..." + if [[ $(ls -ldn "$DIR" | awk '{print $3}') -eq "$USR" ]] && [[ $(ls -ldn "$DIR" | awk '{print $4}') -eq "$GRP" ]]; then + echo "> OK" + IS_OWNER_CHECK_PASSED=true + else + echo "...ownership check failed" + if [ "$CREATE" = false ]; then + EXIT_CODE=1 + fi + fi + fi else - if [ "$1" = "--create" ]; then - echo "Create and chown: user ${USR} group ${GRP} dir ${DIR}" - mkdir -p "$DIR" && sudo chown -R "$USR":"$GRP" "$DIR" - else - echo "$MESSAGE FAILED" + echo "...does not exist" + if [ "$CREATE" = false ]; then EXIT_CODE=1 fi fi + + # create/chown section + if [ "$CREATE" = true ]; then + if [ "$IS_EXIST_CHECK_PASSED" = false ]; then + echo "...will create dir ${DIR}" + if [ "$SKIP_CHOWN" = false ]; then + echo "...will change ownership to user ${USR} group ${GRP} for dir ${DIR}" + mkdir -p "$DIR" && sudo chown -R "$USR":"$GRP" "$DIR" && echo "> OK" + else + mkdir -p "$DIR" && echo "> OK" + fi + elif [ "$IS_OWNER_CHECK_PASSED" = false ] && [ "$SKIP_CHOWN" = false ]; then + echo "...will change ownership to user ${USR} group ${GRP} for dir ${DIR}" + sudo chown -R "$USR":"$GRP" "$DIR" && echo "> OK" + fi + fi + done < <(echo "$PERMISSION_LIST") return $EXIT_CODE } diff --git a/docker/docker-check-log-folders.sh b/docker/docker-check-log-folders.sh index e293968a69..6122f3d2c1 100755 --- a/docker/docker-check-log-folders.sh +++ b/docker/docker-check-log-folders.sh @@ -17,5 +17,12 @@ set -e source compose-utils.sh -checkFolders || exit $? -echo "OK" +if checkFolders "$@" ; then + echo "------" + echo "All checks have passed" +else + CHECK_EXIT_CODE=$? + echo "------" + echo "Some checks did not pass - check the output" + exit $CHECK_EXIT_CODE +fi \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 69b7722459..1cee5ad5ad 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,7 @@ version: '3.0' services: zookeeper: restart: always - image: "zookeeper:3.8.0" + image: "zookeeper:3.8.1" ports: - "2181" environment: diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 098ffabb31..ed66d4e156 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -17,4 +17,4 @@ set -e source compose-utils.sh -checkFolders --create +checkFolders --create "$@" diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index da09684d4f..aa68a2252f 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -53,8 +53,6 @@ ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -checkFolders --create || exit $? - if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then COMPOSE_ARGS="\ diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 8b380f199b..0d256abcf6 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -31,8 +31,6 @@ ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? -checkFolders --create || exit $? - COMPOSE_ARGS="\ -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index eca5d34957..05293e475e 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -46,8 +46,6 @@ ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -checkFolders --create || exit $? - COMPOSE_ARGS_PULL="\ -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 1d7f111e9a..f7d0eda841 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -70,8 +70,6 @@ queue: stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" - # Statistics printing interval for EDQS - print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:300000}" kafka: # Kafka Bootstrap nodes in "host:port" format diff --git a/ui-ngx/package.json b/ui-ngx/package.json index b2e2dc18c3..1536858de8 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -28,6 +28,7 @@ "@flowjs/ngx-flow": "18.0.1", "@geoman-io/leaflet-geoman-free": "2.17.0", "@iplab/ngx-color-picker": "^18.0.1", + "@maplibre/maplibre-gl-leaflet": "^0.0.22", "@mat-datetimepicker/core": "~14.0.0", "@mdi/svg": "^7.4.47", "@messageformat/core": "^3.4.0", @@ -64,6 +65,7 @@ "leaflet.gridlayer.googlemutant": "0.14.1", "leaflet.markercluster": "1.5.3", "libphonenumber-js": "^1.11.15", + "maplibre-gl": "^4.7.1", "marked": "~12.0.2", "moment": "^2.30.1", "moment-timezone": "^0.5.45", diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet/leaflet-tb.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet/leaflet-tb.ts index 06d49d0343..df02c46482 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet/leaflet-tb.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/leaflet/leaflet-tb.ts @@ -17,6 +17,7 @@ import L, { TB } from 'leaflet'; import { guid, isNotEmptyStr } from '@core/utils'; import 'leaflet-providers'; +import '@maplibre/maplibre-gl-leaflet'; import '@geoman-io/leaflet-geoman-free'; import 'leaflet.markercluster'; import { MatIconRegistry } from '@angular/material/icon'; @@ -211,15 +212,19 @@ class LayersControl extends SidebarPaneControl { input.on('click', (e: JQuery.MouseEventBase) => { e.stopPropagation(); - layers.forEach((other) => { - if (other.layer === layerData.layer) { - map.addLayer(other.layer); - map.attributionControl.setPrefix(other.attributionPrefix); - } else { - map.removeLayer(other.layer); + if (!map.hasLayer(layerData.layer)) { + map.addLayer(layerData.layer); + map.attributionControl.setPrefix(layerData.attributionPrefix); + if (layerData.onAdd) { + layerData.onAdd(); } - }); - map.fire('baselayerchange', { layer: layerData.layer }); + layers.forEach((other) => { + if (other.layer !== layerData.layer) { + map.removeLayer(other.layer); + } + }); + map.fire('baselayerchange', { layer: layerData.layer }); + } }); item.on('dblclick', (e) => { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-layer.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-layer.ts index 304100828b..655cc26374 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-layer.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-layer.ts @@ -26,17 +26,34 @@ import { HereMapLayerSettings, MapLayerSettings, MapProvider, - OpenStreetMapLayerSettings, + OpenStreetMapLayerSettings, ReferenceLayerType, TencentMapLayerSettings } from '@shared/models/widget/maps/map.models'; import { WidgetContext } from '@home/models/widget-component.models'; import { DeepPartial } from '@shared/models/common'; import { mergeDeep } from '@core/utils'; -import { Observable, of, switchMap } from 'rxjs'; +import { Observable, of, shareReplay, switchMap } from 'rxjs'; import { CustomTranslatePipe } from '@shared/pipe/custom-translate.pipe'; import L from 'leaflet'; import { catchError, map } from 'rxjs/operators'; import { ResourcesService } from '@core/services/resources.service'; +import { StyleSpecification, VectorSourceSpecification } from '@maplibre/maplibre-gl-style-spec'; +import { ResourceType } from 'maplibre-gl'; + +const referenceLayerStyleUrlMap = new Map( + [ + [ReferenceLayerType.openstreetmap_hybrid, '/assets/map/openstreetmap_hybrid_reference_style.json'], + [ReferenceLayerType.world_edition_hybrid, '/assets/map/world_edition_hybrid_reference_style.json'] + ] +); + +const referenceLayerCache = new Map>(); + +interface TbMapLayerData { + layer: L.Layer; + attribution: boolean; + onAdd?: () => void; +} export abstract class TbMapLayer { @@ -65,19 +82,23 @@ export abstract class TbMapLayer { } public loadLayer(theMap: L.Map): Observable { - return this.createLayer().pipe( - switchMap((layer) => { - if (layer) { - return this.createLayer().pipe( - map((mini) => { - if (mini) { - const attribution = layer.getAttribution(); - const attributionPrefix = attribution ? theMap.attributionControl.options.prefix as string : null; + return this.generateLayer().pipe( + switchMap((layerData) => { + if (layerData) { + return this.generateLayer().pipe( + map((miniLayerData) => { + if (miniLayerData) { + const attributionPrefix = layerData.attribution ? theMap.attributionControl.options.prefix as string : null; return { title: this.title(), attributionPrefix: attributionPrefix, - layer, - mini + layer: layerData.layer, + mini: miniLayerData.layer, + onAdd: () => { + if (layerData.onAdd) { + layerData.onAdd(); + } + } }; } else { return null; @@ -91,6 +112,69 @@ export abstract class TbMapLayer { ); } + private generateLayer(): Observable { + return this.createLayer().pipe( + switchMap((baseLayer) => { + if (baseLayer) { + if (this.settings.referenceLayer) { + return this.loadReferenceLayer(this.settings.referenceLayer).pipe( + map((referenceLayer) => { + if (referenceLayer) { + const layer = L.featureGroup(); + baseLayer.addTo(layer); + referenceLayer.addTo(layer); + return { + layer, + attribution: !!baseLayer.getAttribution() || !!referenceLayer.getAttribution(), + onAdd: () => { + (referenceLayer as any)._update(); + } + }; + } else { + return { + layer: baseLayer, + attribution: !!baseLayer.getAttribution() + }; + } + })); + } else { + return of({ + layer: baseLayer, + attribution: !!baseLayer.getAttribution() + }); + } + } else { + return of(null); + } + } + )); + } + + private loadReferenceLayer(referenceLayer: ReferenceLayerType): Observable { + let spec$ = referenceLayerCache.get(referenceLayer); + if (!spec$) { + const styleUrl = referenceLayerStyleUrlMap.get(referenceLayer); + spec$ = this.ctx.http.get(styleUrl).pipe( + shareReplay({ + bufferSize: 1, + refCount: true + }) + ); + referenceLayerCache.set(referenceLayer, spec$); + } + return spec$.pipe( + map(spec => { + const sourceSpec = (spec.sources['esri'] as VectorSourceSpecification); + const attribution = sourceSpec.attribution; + const gl = L.maplibreGL({ + style: spec, + }); + gl.options.attribution = attribution; + return gl; + }) + ); + } + private title(): string { const customTranslate = this.ctx.$injector.get(CustomTranslatePipe); if (this.settings.label) { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map.scss b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map.scss index 592610f8b2..985bdc17f5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/map.scss +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/map.scss @@ -35,6 +35,10 @@ div.tb-widget .tb-widget-content.tb-no-interaction { .tb-map-container { flex-direction: column; + .leaflet-gl-layer.maplibregl-map { + position: relative; + z-index: 1; + } } .tb-map-layout { @@ -47,6 +51,10 @@ div.tb-widget .tb-widget-content.tb-no-interaction { .tb-map { position: relative; flex: 1; + .leaflet-control-attribution { + font-size: 0.6rem; + background: rgba(255,255,255,0.5); + } &.leaflet-touch { .leaflet-bar { border: 1px solid rgba(0,0,0,0.38); diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-row.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-row.component.ts index 7112a0c3c2..392c9f9565 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-row.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-row.component.ts @@ -123,7 +123,8 @@ export class MapLayerRowComponent implements ControlValueAccessor, OnInit { provider: [null, [Validators.required]], layerType: [null, [Validators.required]], tileUrl: [null, [Validators.required]], - apiKey: [null, [Validators.required]] + apiKey: [null, [Validators.required]], + referenceLayer: [null, []] }); this.layerFormGroup.valueChanges.pipe( takeUntilDestroyed(this.destroyRef) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-settings-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-settings-panel.component.html index c30033f7e1..cf4c4b90c4 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-settings-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/map/map-layer-settings-panel.component.html @@ -82,6 +82,17 @@ +
+
widgets.maps.layer.reference.reference-layer
+ + + {{ 'widgets.maps.layer.reference.no-layer' | translate }} + + {{ referenceLayerTypeTranslationMap.get(layer) | translate }} + + + +