From 7c174de71da9decd27a38804e7dbafdb358567cc Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 21 Sep 2023 17:48:53 +0200 Subject: [PATCH 01/36] BaseController: log experience improvement. use a child class logger instead of abstract class controller --- .../org/thingsboard/server/controller/BaseController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 0af55d7c00..365f1fc379 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -21,7 +21,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; -import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.MediaType; @@ -174,10 +174,11 @@ import static org.thingsboard.server.common.data.query.EntityKeyType.ENTITY_FIEL import static org.thingsboard.server.controller.UserController.YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION; import static org.thingsboard.server.dao.service.Validator.validateId; -@Slf4j @TbCoreComponent public abstract class BaseController { + private final Logger log = org.slf4j.LoggerFactory.getLogger(getClass()); + /*Swagger UI description*/ @Autowired From 28b23ba9b40e658383b38b0d9a950109b2fd8eaa Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 29 Sep 2023 11:49:41 +0200 Subject: [PATCH 02/36] fixed AlarmTypes redis cache serialization --- .../server/common/data/EntitySubtype.java | 79 ++----------------- .../server/common/data/page/PageData.java | 2 + .../dao/service/TbCacheSerializationTest.java | 50 ++++++++++++ 3 files changed, 60 insertions(+), 71 deletions(-) create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/TbCacheSerializationTest.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java index a776635c61..b2e78319dd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java @@ -15,80 +15,17 @@ */ package org.thingsboard.server.common.data; +import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; -public class EntitySubtype { +import java.io.Serializable; - private static final long serialVersionUID = 8057240243059922101L; - - private TenantId tenantId; - private EntityType entityType; - private String type; - - public EntitySubtype() { - super(); - } - - public EntitySubtype(TenantId tenantId, EntityType entityType, String type) { - this.tenantId = tenantId; - this.entityType = entityType; - this.type = type; - } - - public TenantId getTenantId() { - return tenantId; - } - - public void setTenantId(TenantId tenantId) { - this.tenantId = tenantId; - } - - public EntityType getEntityType() { - return entityType; - } - - public void setEntityType(EntityType entityType) { - this.entityType = entityType; - } - - public String getType() { - return type; - } +@Data +public class EntitySubtype implements Serializable { - public void setType(String type) { - this.type = type; - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - EntitySubtype that = (EntitySubtype) o; - - if (tenantId != null ? !tenantId.equals(that.tenantId) : that.tenantId != null) return false; - if (entityType != that.entityType) return false; - return type != null ? type.equals(that.type) : that.type == null; - - } - - @Override - public int hashCode() { - int result = tenantId != null ? tenantId.hashCode() : 0; - result = 31 * result + (entityType != null ? entityType.hashCode() : 0); - result = 31 * result + (type != null ? type.hashCode() : 0); - return result; - } - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("EntitySubtype{"); - sb.append("tenantId=").append(tenantId); - sb.append(", entityType=").append(entityType); - sb.append(", type='").append(type).append('\''); - sb.append('}'); - return sb.toString(); - } + private static final long serialVersionUID = 8057240243059922101L; + private final TenantId tenantId; + private final EntityType entityType; + private final String type; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java index 6eb9218956..dd49aa7d97 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import java.io.Serializable; import java.util.Collections; @@ -27,6 +28,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @ApiModel +@EqualsAndHashCode public class PageData implements Serializable { public static final PageData EMPTY_PAGE_DATA = new PageData<>(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/TbCacheSerializationTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/TbCacheSerializationTest.java new file mode 100644 index 0000000000..74306e6234 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/TbCacheSerializationTest.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +@DaoSqlTest +public class TbCacheSerializationTest extends AbstractServiceTest { + + @Autowired + TbTransactionalCache> alarmTypesCache; + + @Test + public void AlarmTypesSerializationTest() { + var typesCount = 13; + TenantId tenantId = new TenantId(UUID.randomUUID()); + List types = new ArrayList<>(typesCount); + for (int i = 0; i < typesCount; i++) { + types.add(new EntitySubtype(tenantId, EntityType.ALARM, "alarm_type_" + i)); + } + PageData alarmTypesPage = new PageData<>(types, 1, typesCount, false); + alarmTypesCache.put(tenantId, alarmTypesPage); + PageData foundAlarmTypes = alarmTypesCache.get(tenantId).get(); + Assert.assertEquals(alarmTypesPage, foundAlarmTypes); + } +} From f001b2588d35a62e1a8a195e7f6d37ee74865b5d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 2 Oct 2023 23:03:15 +0200 Subject: [PATCH 03/36] leave toString as-is --- .../thingsboard/server/common/data/EntitySubtype.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java index b2e78319dd..93454f6656 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntitySubtype.java @@ -28,4 +28,14 @@ public class EntitySubtype implements Serializable { private final TenantId tenantId; private final EntityType entityType; private final String type; + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("EntitySubtype{"); + sb.append("tenantId=").append(tenantId); + sb.append(", entityType=").append(entityType); + sb.append(", type='").append(type).append('\''); + sb.append('}'); + return sb.toString(); + } } From 6ec4b8cf7253f85b11c3ec5376285f0281884b53 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 26 Sep 2023 13:17:52 +0300 Subject: [PATCH 04/36] TTL for queue stats and RE exceptions --- .../controller/TenantProfileController.java | 2 + .../DefaultRuleEngineStatisticsService.java | 28 +++--- .../controller/BaseQueueControllerTest.java | 95 +++++++++++++++++++ .../dao/usagerecord/ApiLimitService.java | 5 + .../DefaultTenantProfileConfiguration.java | 2 + .../usagerecord/DefaultApiLimitService.java | 34 +++++-- ...enant-profile-configuration.component.html | 28 ++++++ ...-tenant-profile-configuration.component.ts | 2 + ui-ngx/src/app/shared/models/tenant.model.ts | 4 + .../assets/locale/locale.constant-en_US.json | 6 ++ 10 files changed, 186 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java index a7fe973ec0..6fbee00f7f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java @@ -151,6 +151,8 @@ public class TenantProfileController extends BaseController { " \"defaultStorageTtlDays\": 0,\n" + " \"alarmsTtlDays\": 0,\n" + " \"rpcTtlDays\": 0,\n" + + " \"queueStatsTtlDays\": 0,\n" + + " \"ruleEngineExceptionsTtlDays\": 0,\n" + " \"warnThreshold\": 0\n" + " }\n" + " },\n" + diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 1b5bfd56d1..30ffb4c8ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.stats; import com.google.common.util.concurrent.FutureCallback; import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.asset.Asset; @@ -26,7 +27,9 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -37,6 +40,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -44,9 +48,11 @@ import java.util.stream.Collectors; @TbRuleEngineComponent @Service @Slf4j +@RequiredArgsConstructor public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService { public static final String TB_SERVICE_QUEUE = "TbServiceQueue"; + public static final String RULE_ENGINE_EXCEPTION = "ruleEngineException"; public static final FutureCallback CALLBACK = new FutureCallback() { @Override public void onSuccess(@Nullable Integer result) { @@ -61,16 +67,10 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS private final TbServiceInfoProvider serviceInfoProvider; private final TelemetrySubscriptionService tsService; - private final Lock lock = new ReentrantLock(); private final AssetService assetService; - private final ConcurrentMap tenantQueueAssets; - - public DefaultRuleEngineStatisticsService(TelemetrySubscriptionService tsService, TbServiceInfoProvider serviceInfoProvider, AssetService assetService) { - this.tsService = tsService; - this.serviceInfoProvider = serviceInfoProvider; - this.assetService = assetService; - this.tenantQueueAssets = new ConcurrentHashMap<>(); - } + private final ApiLimitService apiLimitService; + private final Lock lock = new ReentrantLock(); + private final ConcurrentMap tenantQueueAssets = new ConcurrentHashMap<>(); @Override public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { @@ -84,7 +84,9 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS .map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get()))) .collect(Collectors.toList()); if (!tsList.isEmpty()) { - tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, CALLBACK); + long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getQueueStatsTtlDays); + ttl = TimeUnit.DAYS.toSeconds(ttl); + tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, ttl, CALLBACK); } } } catch (Exception e) { @@ -95,8 +97,10 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { try { - TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry("ruleEngineException", e.toJsonString())); - tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), CALLBACK); + TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry(RULE_ENGINE_EXCEPTION, e.toJsonString())); + long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getRuleEngineExceptionsTtlDays); + ttl = TimeUnit.DAYS.toSeconds(ttl); + tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK); } catch (Exception e2) { if (!"Asset is referencing to non-existent tenant!".equalsIgnoreCase(e2.getMessage())) { log.debug("[{}] Failed to store the statistics", tenantId, e2); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 465ec37bb9..9da8c8f496 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -18,6 +18,12 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -26,13 +32,45 @@ import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.msg.queue.RuleEngineException; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; +import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult; +import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; +import org.thingsboard.server.service.stats.RuleEngineStatisticsService; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @DaoSqlTest public class BaseQueueControllerTest extends AbstractControllerTest { + @Autowired + private RuleEngineStatisticsService ruleEngineStatisticsService; + @Autowired + private StatsFactory statsFactory; + @SpyBean + private TimeseriesDao timeseriesDao; + @Autowired + private AssetService assetService; + @Test public void testQueueWithServiceTypeRE() throws Exception { loginSysAdmin(); @@ -93,4 +131,61 @@ public class BaseQueueControllerTest extends AbstractControllerTest { .andExpect(status().isOk()); } + @Test + public void testQueueStatsTtl() throws ThingsboardException { + Queue queue = new Queue(); + queue.setName("Test-1"); + queue.setTenantId(TenantId.SYS_TENANT_ID); + + TbRuleEngineProcessingResult testProcessingResult = Mockito.mock(TbRuleEngineProcessingResult.class); + TbProtoQueueMsg msg = new TbProtoQueueMsg<>(UUID.randomUUID(), + TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .build()); + when(testProcessingResult.getSuccessMap()).thenReturn(Stream.generate(() -> msg) + .limit(5).collect(Collectors.toConcurrentMap(m -> UUID.randomUUID(), m -> m))); + when(testProcessingResult.getFailedMap()).thenReturn(Stream.generate(() -> msg) + .limit(5).collect(Collectors.toConcurrentMap(m -> UUID.randomUUID(), m -> m))); + when(testProcessingResult.getPendingMap()).thenReturn(new ConcurrentHashMap<>()); + RuleEngineException ruleEngineException = new RuleEngineException("Test Exception"); + when(testProcessingResult.getExceptionsMap()).thenReturn(new ConcurrentHashMap<>(Map.of( + tenantId, ruleEngineException + ))); + + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + testStats.log(testProcessingResult, true); + + int queueStatsTtlDays = 14; + int ruleEngineExceptionsTtlDays = 7; + updateDefaultTenantProfileConfig(profileConfiguration -> { + profileConfiguration.setQueueStatsTtlDays(queueStatsTtlDays); + profileConfiguration.setRuleEngineExceptionsTtlDays(ruleEngineExceptionsTtlDays); + }); + ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); + + Asset serviceAsset = assetService.findAssetsByTenantIdAndType(tenantId, TB_SERVICE_QUEUE, new PageLink(100)).getData() + .stream().filter(asset -> asset.getName().startsWith(queue.getName())) + .findFirst().get(); + + ArgumentCaptor ttlCaptor = ArgumentCaptor.forClass(Long.class); + verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS) && + tsKvEntry.getLongValue().get().equals(5L); + }), ttlCaptor.capture()); + verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.FAILED_MSGS) && + tsKvEntry.getLongValue().get().equals(5L); + }), ttlCaptor.capture()); + assertThat(ttlCaptor.getAllValues()).allSatisfy(usedTtl -> { + assertThat(usedTtl).isEqualTo(TimeUnit.DAYS.toSeconds(queueStatsTtlDays)); + }); + + verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { + return tsKvEntry.getKey().equals(DefaultRuleEngineStatisticsService.RULE_ENGINE_EXCEPTION) && + tsKvEntry.getJsonValue().get().equals(ruleEngineException.toJsonString()); + }), ttlCaptor.capture()); + assertThat(ttlCaptor.getValue()).isEqualTo(TimeUnit.DAYS.toSeconds(ruleEngineExceptionsTtlDays)); + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiLimitService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiLimitService.java index cd9524e1b1..69e366327d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiLimitService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiLimitService.java @@ -17,9 +17,14 @@ package org.thingsboard.server.dao.usagerecord; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; + +import java.util.function.Function; public interface ApiLimitService { boolean checkEntitiesLimit(TenantId tenantId, EntityType entityType); + long getLimit(TenantId tenantId, Function extractor); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index bd2b80f10a..77a7fa8417 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -82,6 +82,8 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private int defaultStorageTtlDays; private int alarmsTtlDays; private int rpcTtlDays; + private int queueStatsTtlDays; + private int ruleEngineExceptionsTtlDays; private double warnThreshold; diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/DefaultApiLimitService.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/DefaultApiLimitService.java index d5d19aa453..56bb096b5c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/DefaultApiLimitService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/DefaultApiLimitService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.usagerecord; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,6 +28,8 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import java.util.function.Function; + @Service @RequiredArgsConstructor public class DefaultApiLimitService implements ApiLimitService { @@ -36,16 +39,31 @@ public class DefaultApiLimitService implements ApiLimitService { @Override public boolean checkEntitiesLimit(TenantId tenantId, EntityType entityType) { - DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration(); - long limit = profileConfiguration.getEntitiesLimit(entityType); - if (limit > 0) { - EntityTypeFilter filter = new EntityTypeFilter(); - filter.setEntityType(entityType); - long currentCount = entityService.countEntitiesByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), new EntityCountQuery(filter)); - return currentCount < limit; - } else { + long limit = getLimit(tenantId, profileConfiguration -> profileConfiguration.getEntitiesLimit(entityType)); + if (limit <= 0) { return true; } + + EntityTypeFilter filter = new EntityTypeFilter(); + filter.setEntityType(entityType); + long currentCount = entityService.countEntitiesByQuery(tenantId, new CustomerId(EntityId.NULL_UUID), new EntityCountQuery(filter)); + return currentCount < limit; + } + + @Override + public long getLimit(TenantId tenantId, Function extractor) { + if (tenantId == null || tenantId.isSysTenantId()) { + return 0L; + } + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + if (tenantProfile == null) { + throw new IllegalArgumentException("Tenant profile not found for tenant " + tenantId); + } + Number value = extractor.apply(tenantProfile.getDefaultProfileConfiguration()); + if (value == null) { + return 0L; + } + return Math.max(0, value.longValue()); } } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index c2872d4145..2a4c59a7a2 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -262,6 +262,34 @@ +
+ + tenant-profile.queue-stats-ttl-days + + + {{ 'tenant-profile.queue-stats-ttl-days-required' | translate}} + + + {{ 'tenant-profile.queue-stats-ttl-days-range' | translate}} + + + + + tenant-profile.rule-engine-exceptions-ttl-days + + + {{ 'tenant-profile.rule-engine-exceptions-ttl-days-required' | translate}} + + + {{ 'tenant-profile.rule-engine-exceptions-ttl-days-days-range' | translate}} + + + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index a4915f924b..19eb13540b 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -90,6 +90,8 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA defaultStorageTtlDays: [null, [Validators.required, Validators.min(0)]], alarmsTtlDays: [null, [Validators.required, Validators.min(0)]], rpcTtlDays: [null, [Validators.required, Validators.min(0)]], + queueStatsTtlDays: [null, [Validators.required, Validators.min(0)]], + ruleEngineExceptionsTtlDays: [null, [Validators.required, Validators.min(0)]], tenantServerRestLimitsConfiguration: [null, []], customerServerRestLimitsConfiguration: [null, []], maxWsSessionsPerTenant: [null, [Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 9074d599f5..d781dad439 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -76,6 +76,8 @@ export interface DefaultTenantProfileConfiguration { defaultStorageTtlDays: number; alarmsTtlDays: number; rpcTtlDays: number; + queueStatsTtlDays: number; + ruleEngineExceptionsTtlDays: number; } export type TenantProfileConfigurations = DefaultTenantProfileConfiguration; @@ -124,6 +126,8 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan defaultStorageTtlDays: 0, alarmsTtlDays: 0, rpcTtlDays: 0, + queueStatsTtlDays: 0, + ruleEngineExceptionsTtlDays: 0 }; configuration = {...defaultConfiguration, type: TenantProfileType.DEFAULT}; break; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0867f21114..876fb19afb 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4003,6 +4003,12 @@ "rpc-ttl-days": "RPC TTL days", "rpc-ttl-days-required": "RPC TTL days required", "rpc-ttl-days-days-range": "RPC TTL days can't be negative", + "queue-stats-ttl-days": "Queue stats TTL days", + "queue-stats-ttl-days-required": "Queue stats TTL days required", + "queue-stats-ttl-days-range": "Queue stats TTL days can't be negative", + "rule-engine-exceptions-ttl-days": "Rule Engine exceptions TTL days", + "rule-engine-exceptions-ttl-days-required": "Rule Engine exceptions TTL days required", + "rule-engine-exceptions-ttl-days-range": "Rule Engine exceptions TTL days can't be negative", "max-rule-node-executions-per-message": "Rule node per message executions maximum number", "max-rule-node-executions-per-message-required": "MRule node per message executions maximum number is required.", "max-rule-node-executions-per-message-range": "Rule node per message executions maximum number can't be negative", From 69349d5075ea6a8b5603817b441cd45464c47c47 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 3 Oct 2023 12:04:06 +0300 Subject: [PATCH 05/36] TTL for system info --- .../service/system/DefaultSystemInfoService.java | 10 ++++++++-- application/src/main/resources/thingsboard.yml | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 80d4cede03..b2412d62d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ProtocolStringList; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -93,6 +94,11 @@ public class DefaultSystemInfoService extends TbApplicationEventListener telemetry) { ApiUsageState apiUsageState = apiUsageStateClient.getApiUsageState(TenantId.SYS_TENANT_ID); - telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, apiUsageState.getId(), telemetry, CALLBACK); + telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, apiUsageState.getId(), telemetry, systemInfoTtlSeconds, CALLBACK); } private List getSystemData(ServiceInfo serviceInfo) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d1e5113449..9fda2275f8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1326,6 +1326,11 @@ metrics: timer: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + system_info: + # Persist frequency of system info (CPU, memory usage, etc.) in seconds + persist_frequency: "${METRICS_SYSTEM_INFO_PERSIST_FREQUENCY_SECONDS:60}" + # TTL in days for system info timeseries + ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" vc: # Pool size for handling export tasks From 12d2c26279b1bf687734ba48a29f734e3ecac18b Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 3 Oct 2023 16:12:53 +0300 Subject: [PATCH 06/36] Rule Engine stats: exception message truncation --- .../DefaultRuleEngineStatisticsService.java | 6 ++- .../src/main/resources/thingsboard.yml | 1 + .../controller/BaseQueueControllerTest.java | 47 ++++++++++++++++++- .../server/common/data/StringUtils.java | 13 +++++ .../server/common/data/StringUtilsTest.java | 11 +++++ .../common/msg/queue/RuleEngineException.java | 16 +++++-- .../common/msg/queue/RuleNodeException.java | 4 +- .../thingsboard/common/util/JacksonUtil.java | 27 ++++------- .../server/dao/event/BaseEventService.java | 8 +--- 9 files changed, 100 insertions(+), 33 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java index 30ffb4c8ca..d8d86922cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java +++ b/application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.FutureCallback; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; @@ -72,6 +73,9 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS private final Lock lock = new ReentrantLock(); private final ConcurrentMap tenantQueueAssets = new ConcurrentHashMap<>(); + @Value("${queue.rule-engine.stats.max-error-message-length:4096}") + private int maxErrorMessageLength; + @Override public void reportQueueStats(long ts, TbRuleEngineConsumerStats ruleEngineStats) { String queueName = ruleEngineStats.getQueueName(); @@ -97,7 +101,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS }); ruleEngineStats.getTenantExceptions().forEach((tenantId, e) -> { try { - TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry(RULE_ENGINE_EXCEPTION, e.toJsonString())); + TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry(RULE_ENGINE_EXCEPTION, e.toJsonString(maxErrorMessageLength))); long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getRuleEngineExceptionsTtlDays); ttl = TimeUnit.DAYS.toSeconds(ttl); tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9fda2275f8..6b11369639 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1248,6 +1248,7 @@ queue: stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" queues: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java index 9da8c8f496..845ac49f15 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java @@ -16,15 +16,19 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.queue.ProcessingStrategy; @@ -48,10 +52,13 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; @@ -60,6 +67,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE; @DaoSqlTest +@TestPropertySource(properties = { + "queue.rule-engine.stats.max-error-message-length=100" +}) public class BaseQueueControllerTest extends AbstractControllerTest { @Autowired @@ -183,9 +193,44 @@ public class BaseQueueControllerTest extends AbstractControllerTest { verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> { return tsKvEntry.getKey().equals(DefaultRuleEngineStatisticsService.RULE_ENGINE_EXCEPTION) && - tsKvEntry.getJsonValue().get().equals(ruleEngineException.toJsonString()); + tsKvEntry.getJsonValue().get().equals(ruleEngineException.toJsonString(0)); }), ttlCaptor.capture()); assertThat(ttlCaptor.getValue()).isEqualTo(TimeUnit.DAYS.toSeconds(ruleEngineExceptionsTtlDays)); } + @Test + public void testRuleEngineExceptionTruncation() { + Queue queue = new Queue(); + queue.setName("Test-2"); + queue.setTenantId(TenantId.SYS_TENANT_ID); + + TbRuleEngineProcessingResult testProcessingResult = Mockito.mock(TbRuleEngineProcessingResult.class); + when(testProcessingResult.getSuccessMap()).thenReturn(new ConcurrentHashMap<>()); + when(testProcessingResult.getFailedMap()).thenReturn(new ConcurrentHashMap<>()); + when(testProcessingResult.getPendingMap()).thenReturn(new ConcurrentHashMap<>()); + + String largeExceptionMessage = RandomStringUtils.randomAlphabetic(150); + RuleEngineException ruleEngineException = new RuleEngineException(largeExceptionMessage); + when(testProcessingResult.getExceptionsMap()).thenReturn(new ConcurrentHashMap<>(Map.of( + tenantId, ruleEngineException + ))); + + TbRuleEngineConsumerStats testStats = new TbRuleEngineConsumerStats(queue, statsFactory); + testStats.log(testProcessingResult, true); + ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats); + + AtomicReference reExceptionTsKvEntryCaptor = new AtomicReference<>(); + verify(timeseriesDao).save(eq(tenantId), any(), argThat(tsKvEntry -> { + if (tsKvEntry.getKey().equals(DefaultRuleEngineStatisticsService.RULE_ENGINE_EXCEPTION)) { + reExceptionTsKvEntryCaptor.set(tsKvEntry); + return true; + } + return false; + }), anyLong()); + TsKvEntry reExceptionTsKvEntry = reExceptionTsKvEntryCaptor.get(); + + String finalErrorMessage = JacksonUtil.toJsonNode(reExceptionTsKvEntry.getJsonValue().get()).get("message").asText(); + assertThat(finalErrorMessage).isEqualTo(largeExceptionMessage.substring(0, 100) + "...[truncated 50 symbols]"); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 449f252411..3abbde8401 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.RandomStringUtils; import java.security.SecureRandom; import java.util.Base64; +import java.util.function.Function; import static org.apache.commons.lang3.StringUtils.repeat; @@ -228,4 +229,16 @@ public class StringUtils { return generateSafeToken(DEFAULT_TOKEN_LENGTH); } + public static String truncate(String string, int maxLength) { + return truncate(string, maxLength, n -> "...[truncated " + n + " symbols]"); + } + + public static String truncate(String string, int maxLength, Function truncationMarkerFunc) { + if (string == null || maxLength <= 0 || string.length() <= maxLength) { + return string; + } + int truncatedSymbols = string.length() - maxLength; + return string.substring(0, maxLength) + truncationMarkerFunc.apply(truncatedSymbols); + } + } diff --git a/common/data/src/test/java/org/thingsboard/server/common/data/StringUtilsTest.java b/common/data/src/test/java/org/thingsboard/server/common/data/StringUtilsTest.java index 4cf21baa99..b28d64cd2d 100644 --- a/common/data/src/test/java/org/thingsboard/server/common/data/StringUtilsTest.java +++ b/common/data/src/test/java/org/thingsboard/server/common/data/StringUtilsTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -37,4 +38,14 @@ class StringUtilsTest { assertThat(StringUtils.contains0x00(sample)).isFalse(); } + @Test + void testTruncate() { + int maxLength = 5; + assertThat(StringUtils.truncate(null, maxLength)).isNull(); + assertThat(StringUtils.truncate("", maxLength)).isEmpty(); + assertThat(StringUtils.truncate("123", maxLength)).isEqualTo("123"); + assertThat(StringUtils.truncate("1234567", maxLength)).isEqualTo("12345...[truncated 2 symbols]"); + assertThat(StringUtils.truncate("1234567", 0)).isEqualTo("1234567"); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java index b8c713cc6b..9003579c20 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleEngineException.java @@ -19,17 +19,17 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.StringUtils; @Slf4j public class RuleEngineException extends Exception { protected static final ObjectMapper mapper = new ObjectMapper(); @Getter - private long ts; + private final long ts; public RuleEngineException(String message) { - super(message != null ? message : "Unknown"); - ts = System.currentTimeMillis(); + this(message, null); } public RuleEngineException(String message, Throwable t) { @@ -37,12 +37,18 @@ public class RuleEngineException extends Exception { ts = System.currentTimeMillis(); } - public String toJsonString() { + public String toJsonString(int maxMessageLength) { try { - return mapper.writeValueAsString(mapper.createObjectNode().put("message", getMessage())); + return mapper.writeValueAsString(mapper.createObjectNode() + .put("message", truncateIfNecessary(getMessage(), maxMessageLength))); } catch (JsonProcessingException e) { log.warn("Failed to serialize exception ", e); throw new RuntimeException(e); } } + + protected String truncateIfNecessary(String message, int maxMessageLength) { + return StringUtils.truncate(message, maxMessageLength); + } + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java index 160e2e6f95..4be1875bc1 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -52,14 +52,14 @@ public class RuleNodeException extends RuleEngineException { } } - public String toJsonString() { + public String toJsonString(int maxMessageLength) { try { return mapper.writeValueAsString(mapper.createObjectNode() .put("ruleNodeId", ruleNodeId.toString()) .put("ruleChainId", ruleChainId.toString()) .put("ruleNodeName", ruleNodeName) .put("ruleChainName", ruleChainName) - .put("message", getMessage())); + .put("message", truncateIfNecessary(getMessage(), maxMessageLength))); } catch (JsonProcessingException e) { log.warn("Failed to serialize exception ", e); throw new RuntimeException(e); diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 7c8d174af2..ee2f91ec4d 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -70,8 +70,7 @@ public class JacksonUtil { try { return fromValue != null ? OBJECT_MAPPER.convertValue(fromValue, toValueType) : null; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("The given object value: " - + fromValue + " cannot be converted to " + toValueType, e); + throw new IllegalArgumentException("The given object value cannot be converted to " + toValueType + ": " + fromValue, e); } } @@ -79,8 +78,7 @@ public class JacksonUtil { try { return fromValue != null ? OBJECT_MAPPER.convertValue(fromValue, toValueTypeRef) : null; } catch (IllegalArgumentException e) { - throw new IllegalArgumentException("The given object value: " - + fromValue + " cannot be converted to " + toValueTypeRef, e); + throw new IllegalArgumentException("The given object value cannot be converted to " + toValueTypeRef + ": " + fromValue, e); } } @@ -88,8 +86,7 @@ public class JacksonUtil { try { return string != null ? OBJECT_MAPPER.readValue(string, clazz) : null; } catch (IOException e) { - throw new IllegalArgumentException("The given string value: " - + string + " cannot be transformed to Json object", e); + throw new IllegalArgumentException("The given string value cannot be transformed to Json object: " + string, e); } } @@ -97,8 +94,7 @@ public class JacksonUtil { try { return string != null ? OBJECT_MAPPER.readValue(string, valueTypeRef) : null; } catch (IOException e) { - throw new IllegalArgumentException("The given string value: " - + string + " cannot be transformed to Json object", e); + throw new IllegalArgumentException("The given string value cannot be transformed to Json object: " + string, e); } } @@ -106,8 +102,7 @@ public class JacksonUtil { try { return string != null ? OBJECT_MAPPER.readValue(string, javaType) : null; } catch (IOException e) { - throw new IllegalArgumentException("The given String value: " - + string + " cannot be transformed to Json object", e); + throw new IllegalArgumentException("The given String value cannot be transformed to Json object: " + string, e); } } @@ -115,8 +110,7 @@ public class JacksonUtil { try { return bytes != null ? OBJECT_MAPPER.readValue(bytes, clazz) : null; } catch (IOException e) { - throw new IllegalArgumentException("The given string value: " - + Arrays.toString(bytes) + " cannot be transformed to Json object", e); + throw new IllegalArgumentException("The given string value cannot be transformed to Json object: " + Arrays.toString(bytes), e); } } @@ -124,8 +118,7 @@ public class JacksonUtil { try { return OBJECT_MAPPER.readTree(bytes); } catch (IOException e) { - throw new IllegalArgumentException("The given byte[] value: " - + Arrays.toString(bytes) + " cannot be transformed to Json object", e); + throw new IllegalArgumentException("The given byte[] value cannot be transformed to Json object: " + Arrays.toString(bytes), e); } } @@ -133,8 +126,7 @@ public class JacksonUtil { try { return value != null ? OBJECT_MAPPER.writeValueAsString(value) : null; } catch (JsonProcessingException e) { - throw new IllegalArgumentException("The given Json object value: " - + value + " cannot be transformed to a String", e); + throw new IllegalArgumentException("The given Json object value cannot be transformed to a String: " + value, e); } } @@ -208,8 +200,7 @@ public class JacksonUtil { try { return OBJECT_MAPPER.writeValueAsBytes(value); } catch (JsonProcessingException e) { - throw new IllegalArgumentException("The given Json object value: " - + value + " cannot be transformed to a String", e); + throw new IllegalArgumentException("The given Json object value cannot be transformed to a String: " + value, e); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index c310141a60..5958a43962 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -92,12 +92,8 @@ public class BaseEventService implements EventService { private void truncateField(T event, Function getter, BiConsumer setter) { var str = getter.apply(event); - if (StringUtils.isNotEmpty(str)) { - var length = str.length(); - if (length > maxDebugEventSymbols) { - setter.accept(event, str.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]"); - } - } + str = StringUtils.truncate(str, maxDebugEventSymbols); + setter.accept(event, str); } @Override From 1a3d7bfab0a73c67619781dbecf37f6cd1f0b8a2 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 3 Oct 2023 17:18:11 +0300 Subject: [PATCH 07/36] Revert initial delay for saveCurrentSystemInfo --- .../server/service/system/DefaultSystemInfoService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index b2412d62d1..4b2a3faa2b 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -107,7 +107,7 @@ public class DefaultSystemInfoService extends TbApplicationEventListener Date: Fri, 29 Sep 2023 16:00:39 +0300 Subject: [PATCH 08/36] Remove foreign keys for notification table --- .../main/data/upgrade/3.6.0/schema_update.sql | 26 +++++++++ .../notification/NotificationApiTest.java | 53 +++++++++++++++++++ .../notification/NotificationRuleApiTest.java | 4 +- .../DefaultNotificationRequestService.java | 3 +- .../dao/notification/NotificationDao.java | 4 ++ .../sql/notification/JpaNotificationDao.java | 10 ++++ .../notification/NotificationRepository.java | 6 +++ .../server/dao/user/UserServiceImpl.java | 3 ++ .../resources/sql/schema-entities-idx.sql | 4 ++ .../main/resources/sql/schema-entities.sql | 4 +- 10 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 application/src/main/data/upgrade/3.6.0/schema_update.sql diff --git a/application/src/main/data/upgrade/3.6.0/schema_update.sql b/application/src/main/data/upgrade/3.6.0/schema_update.sql new file mode 100644 index 0000000000..0ad2d97b7a --- /dev/null +++ b/application/src/main/data/upgrade/3.6.0/schema_update.sql @@ -0,0 +1,26 @@ +-- +-- Copyright © 2016-2023 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER TABLE widget_type + ADD COLUMN IF NOT EXISTS tags text[]; + +ALTER TABLE api_usage_state ADD COLUMN IF NOT EXISTS tbel_exec varchar(32); +UPDATE api_usage_state SET tbel_exec = js_exec WHERE tbel_exec IS NULL; + +ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_request_id; +ALTER TABLE notification DROP CONSTRAINT IF EXISTS fk_notification_recipient_id; +CREATE INDEX IF NOT EXISTS idx_notification_notification_request_id ON notification(request_id); +CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_request(tenant_id); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 1b3fce3c3c..54b3079465 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -25,9 +25,11 @@ import org.mockito.ArgumentCaptor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.client.RestTemplate; import org.thingsboard.rule.engine.api.NotificationCenter; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.NotificationRequestId; import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.TenantId; @@ -62,6 +64,7 @@ import org.thingsboard.server.common.data.notification.template.NotificationTemp import org.thingsboard.server.common.data.notification.template.SlackDeliveryMethodNotificationTemplate; import org.thingsboard.server.common.data.notification.template.SmsDeliveryMethodNotificationTemplate; import org.thingsboard.server.common.data.notification.template.WebDeliveryMethodNotificationTemplate; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.notification.DefaultNotifications; import org.thingsboard.server.dao.notification.NotificationDao; @@ -74,6 +77,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -86,6 +90,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @Slf4j @@ -279,6 +284,49 @@ public class NotificationApiTest extends AbstractNotificationApiTest { assertThat(getMyNotifications(false, 10)).size().isZero(); } + @Test + public void whenTenantIsDeleted_thenDeleteNotifications() throws Exception { + createDifferentTenant(); + NotificationTarget target = createNotificationTarget(savedDifferentTenantUser.getId()); + int notificationsCount = 20; + for (int i = 0; i < notificationsCount; i++) { + NotificationRequest request = submitNotificationRequest(target.getId(), "Test " + i, NotificationDeliveryMethod.WEB); + awaitNotificationRequest(request.getId()); + } + List requests = notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(100)).getData(); + assertThat(requests).size().isEqualTo(notificationsCount); + for (NotificationRequest request : requests) { + List notifications = notificationDao.findByRequestId(differentTenantId, request.getId(), new PageLink(100)).getData(); + assertThat(notifications).size().isNotZero(); + } + + deleteDifferentTenant(); + + assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(1)).getTotalElements()) + .isZero(); + for (NotificationRequest request : requests) { + assertThat(notificationDao.findByRequestId(differentTenantId, request.getId(), new PageLink(100)).getTotalElements()) + .isZero(); + } + } + + @Test + public void whenUserIsDeleted_thenDeleteNotifications() throws Exception { + NotificationTarget target = createNotificationTarget(customerUserId); + int notificationsCount = 20; + for (int i = 0; i < notificationsCount; i++) { + NotificationRequest request = submitNotificationRequest(target.getId(), "Test " + i, NotificationDeliveryMethod.WEB); + awaitNotificationRequest(request.getId()); + } + List notifications = notificationDao.findByRecipientIdAndPageLink(tenantId, customerUserId, new PageLink(100)).getData(); + assertThat(notifications).size().isGreaterThanOrEqualTo(notificationsCount); + + doDelete("/api/user/" + customerUserId).andExpect(status().isOk()); + + notifications = notificationDao.findByRecipientIdAndPageLink(tenantId, customerUserId, new PageLink(100)).getData(); + assertThat(notifications).isEmpty(); + } + @Test public void testNotificationUpdatesForSeveralUsers() throws Exception { int usersCount = 150; @@ -692,6 +740,11 @@ public class NotificationApiTest extends AbstractNotificationApiTest { return future.get(30, TimeUnit.SECONDS); } + private NotificationRequestStats awaitNotificationRequest(NotificationRequestId requestId) { + return await().atMost(30, TimeUnit.SECONDS) + .until(() -> getStats(requestId), Objects::nonNull); + } + private void checkFullNotificationsUpdate(UnreadNotificationsUpdate notificationsUpdate, String... expectedNotifications) { assertThat(notificationsUpdate.getNotifications()).extracting(Notification::getText).containsOnly(expectedNotifications); assertThat(notificationsUpdate.getNotifications()).extracting(Notification::getType).containsOnly(DEFAULT_NOTIFICATION_TYPE); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index 0a2854a28d..d08b1a121b 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -456,7 +456,9 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { loginSysAdmin(); notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10), list -> list.size() == 1); + .until(() -> getMyNotifications(true, 10).stream() + .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) + .collect(Collectors.toList()), list -> list.size() == 1); assertThat(notifications).allSatisfy(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME); }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index e951a75c7c..e388c189a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -42,6 +42,7 @@ import java.util.Optional; public class DefaultNotificationRequestService implements NotificationRequestService, EntityDaoService { private final NotificationRequestDao notificationRequestDao; + private final NotificationDao notificationDao; private final NotificationRequestValidator notificationRequestValidator = new NotificationRequestValidator(); @@ -81,10 +82,10 @@ public class DefaultNotificationRequestService implements NotificationRequestSer return notificationRequestDao.findByRuleIdAndOriginatorEntityId(tenantId, ruleId, originatorEntityId); } - // ON DELETE CASCADE is used: notifications for request are deleted as well @Override public void deleteNotificationRequest(TenantId tenantId, NotificationRequestId requestId) { notificationRequestDao.removeById(tenantId, requestId.getId()); + notificationDao.deleteByRequestId(tenantId, requestId); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationDao.java index 1ced714bce..fb3890fcbb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationDao.java @@ -41,4 +41,8 @@ public interface NotificationDao extends Dao { int updateStatusByRecipientId(TenantId tenantId, UserId recipientId, NotificationStatus status); + void deleteByRequestId(TenantId tenantId, NotificationRequestId requestId); + + void deleteByRecipientId(TenantId tenantId, UserId recipientId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationDao.java index 7eeab8c592..21a5ab5c0c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationDao.java @@ -104,6 +104,16 @@ public class JpaNotificationDao extends JpaAbstractDao getEntityClass() { return NotificationEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRepository.java index c08a3c290e..586af7d674 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationRepository.java @@ -61,6 +61,12 @@ public interface NotificationRepository extends JpaRepository userValidator; private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; @@ -255,6 +257,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic UserCredentials userCredentials = userCredentialsDao.findByUserId(tenantId, userId.getId()); userCredentialsDao.removeById(tenantId, userCredentials.getUuidId()); userAuthSettingsDao.removeByUserId(userId); + notificationDao.deleteByRecipientId(tenantId, userId); deleteEntityRelations(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index 675fcd3ec0..68eede5ce4 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -104,6 +104,8 @@ CREATE INDEX IF NOT EXISTS idx_notification_rule_tenant_id_trigger_type_created_ CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id_user_created_time ON notification_request(tenant_id, created_time DESC) WHERE originator_entity_type = 'USER'; +CREATE INDEX IF NOT EXISTS idx_notification_request_tenant_id ON notification_request(tenant_id); + CREATE INDEX IF NOT EXISTS idx_notification_request_rule_id_originator_entity_id ON notification_request(rule_id, originator_entity_id) WHERE originator_entity_type = 'ALARM'; @@ -112,6 +114,8 @@ CREATE INDEX IF NOT EXISTS idx_notification_request_status ON notification_reque CREATE INDEX IF NOT EXISTS idx_notification_id ON notification(id); +CREATE INDEX IF NOT EXISTS idx_notification_notification_request_id ON notification(request_id); + CREATE INDEX IF NOT EXISTS idx_notification_recipient_id_created_time ON notification(recipient_id, created_time DESC); CREATE INDEX IF NOT EXISTS idx_notification_recipient_id_unread ON notification(recipient_id) WHERE status <> 'READ'; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 4df73de9d6..bc49142e44 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -851,8 +851,8 @@ CREATE TABLE IF NOT EXISTS notification_request ( CREATE TABLE IF NOT EXISTS notification ( id UUID NOT NULL, created_time BIGINT NOT NULL, - request_id UUID NULL CONSTRAINT fk_notification_request_id REFERENCES notification_request(id) ON DELETE CASCADE, - recipient_id UUID NOT NULL CONSTRAINT fk_notification_recipient_id REFERENCES tb_user(id) ON DELETE CASCADE, + request_id UUID, + recipient_id UUID NOT NULL, type VARCHAR(50) NOT NULL, subject VARCHAR(255), body VARCHAR(1000) NOT NULL, From c0294e387854de892920ad4540f637912371e01f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 3 Oct 2023 17:29:28 +0300 Subject: [PATCH 09/36] Don't delete notifications when recipient is deleted --- .../service/ttl/AlarmsCleanUpService.java | 1 - .../ttl/NotificationsCleanUpService.java | 4 +-- .../notification/NotificationApiTest.java | 28 +------------------ .../DefaultNotificationRequestService.java | 1 + .../insert/sql/SqlPartitioningRepository.java | 2 +- .../server/dao/user/UserServiceImpl.java | 3 -- 6 files changed, 5 insertions(+), 34 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 2f84767eff..416f95cf96 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -92,7 +92,6 @@ public class AlarmsCleanUpService { while (true) { PageData toRemove = alarmDao.findAlarmsIdsByEndTsBeforeAndTenantId(expirationTime, tenantId, removalBatchRequest); for (AlarmId alarmId : toRemove.getData()) { - relationService.deleteEntityRelations(tenantId, alarmId); Alarm alarm = alarmService.delAlarm(tenantId, alarmId, false).getAlarm(); if (alarm != null) { entityActionService.pushEntityActionToRuleEngine(alarm.getOriginator(), alarm, tenantId, null, ActionType.ALARM_DELETE, null); diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java index 65b04ef0ab..b7e6e57e04 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java @@ -63,8 +63,8 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { if (lastRemovedNotificationTs > 0) { long gap = TimeUnit.MINUTES.toMillis(10); long requestExpTime = lastRemovedNotificationTs - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; - // TODO: double-check this - notificationRequestDao.removeAllByCreatedTimeBefore(requestExpTime); + int removed = notificationRequestDao.removeAllByCreatedTimeBefore(requestExpTime); + log.info("Removed {} outdated notification requests older than {}", removed, requestExpTime); } } diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 54b3079465..21d29833b9 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -90,7 +90,6 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @Slf4j @@ -285,7 +284,7 @@ public class NotificationApiTest extends AbstractNotificationApiTest { } @Test - public void whenTenantIsDeleted_thenDeleteNotifications() throws Exception { + public void whenTenantIsDeleted_thenDeleteNotificationRequests() throws Exception { createDifferentTenant(); NotificationTarget target = createNotificationTarget(savedDifferentTenantUser.getId()); int notificationsCount = 20; @@ -295,36 +294,11 @@ public class NotificationApiTest extends AbstractNotificationApiTest { } List requests = notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(100)).getData(); assertThat(requests).size().isEqualTo(notificationsCount); - for (NotificationRequest request : requests) { - List notifications = notificationDao.findByRequestId(differentTenantId, request.getId(), new PageLink(100)).getData(); - assertThat(notifications).size().isNotZero(); - } deleteDifferentTenant(); assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(1)).getTotalElements()) .isZero(); - for (NotificationRequest request : requests) { - assertThat(notificationDao.findByRequestId(differentTenantId, request.getId(), new PageLink(100)).getTotalElements()) - .isZero(); - } - } - - @Test - public void whenUserIsDeleted_thenDeleteNotifications() throws Exception { - NotificationTarget target = createNotificationTarget(customerUserId); - int notificationsCount = 20; - for (int i = 0; i < notificationsCount; i++) { - NotificationRequest request = submitNotificationRequest(target.getId(), "Test " + i, NotificationDeliveryMethod.WEB); - awaitNotificationRequest(request.getId()); - } - List notifications = notificationDao.findByRecipientIdAndPageLink(tenantId, customerUserId, new PageLink(100)).getData(); - assertThat(notifications).size().isGreaterThanOrEqualTo(notificationsCount); - - doDelete("/api/user/" + customerUserId).andExpect(status().isOk()); - - notifications = notificationDao.findByRecipientIdAndPageLink(tenantId, customerUserId, new PageLink(100)).getData(); - assertThat(notifications).isEmpty(); } @Test diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index e388c189a0..0a11c058e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -98,6 +98,7 @@ public class DefaultNotificationRequestService implements NotificationRequestSer notificationRequestDao.updateById(tenantId, requestId, requestStatus, stats); } + // notifications themselves are left in the database until removed by ttl @Override public void deleteNotificationRequestsByTenantId(TenantId tenantId) { notificationRequestDao.removeByTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java index 60614e8c90..e317e6d7c0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java @@ -140,7 +140,7 @@ public class SqlPartitioningRepository { try { partitions.add(Long.parseLong(partitionTsStr)); } catch (NumberFormatException nfe) { - log.warn("Failed to parse table name: {}", partitionTableName); + log.debug("Failed to parse table name: {}", partitionTableName); } } return partitions; diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 521b1dbd4d..9776e5fe51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -49,7 +49,6 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.IncorrectParameterException; -import org.thingsboard.server.dao.notification.NotificationDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -86,7 +85,6 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic private final UserDao userDao; private final UserCredentialsDao userCredentialsDao; private final UserAuthSettingsDao userAuthSettingsDao; - private final NotificationDao notificationDao; private final DataValidator userValidator; private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; @@ -257,7 +255,6 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic UserCredentials userCredentials = userCredentialsDao.findByUserId(tenantId, userId.getId()); userCredentialsDao.removeById(tenantId, userCredentials.getUuidId()); userAuthSettingsDao.removeByUserId(userId); - notificationDao.deleteByRecipientId(tenantId, userId); deleteEntityRelations(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); From d33d847c0c24de4b2944b577a27d9719fb71b664 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Oct 2023 09:56:23 +0200 Subject: [PATCH 10/36] ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE increased from 4 to 8 --- .../thingsboard/server/actors/service/DefaultActorService.java | 2 +- application/src/main/resources/thingsboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 380da71443..87831cb8f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -74,7 +74,7 @@ public class DefaultActorService extends TbApplicationEventListener Date: Thu, 5 Oct 2023 10:41:29 +0200 Subject: [PATCH 11/36] SQL_ATTRIBUTES_BATCH_SIZE decreased from 10k down to 1k to improve concurrent updates on maximum load --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0b012ba96c..9827a58a53 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -268,8 +268,8 @@ cassandra: sql: # Specify batch size for persisting attribute updates attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:1000}" + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:50}" stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_ATTRIBUTES_VALUE_NO_XSS_VALIDATION:false}" From b6b4124625e2203731953245761f4ade819acb93 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Oct 2023 10:41:43 +0200 Subject: [PATCH 12/36] SQL_TS_LATEST_BATCH_SIZE decreased from 10k down to 1k to improve concurrent updates on maximum load --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9827a58a53..4a3023d48f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -280,8 +280,8 @@ sql: batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution update_by_latest_ts: "${SQL_TS_UPDATE_BY_LATEST_TIMESTAMP:true}" From 6809d374a6e61be66af2356c33722233a9aab403 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Oct 2023 09:56:23 +0200 Subject: [PATCH 13/36] ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE increased from 4 to 8 --- .../thingsboard/server/actors/service/DefaultActorService.java | 2 +- application/src/main/resources/thingsboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 380da71443..87831cb8f6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -74,7 +74,7 @@ public class DefaultActorService extends TbApplicationEventListener Date: Thu, 5 Oct 2023 10:41:29 +0200 Subject: [PATCH 14/36] SQL_ATTRIBUTES_BATCH_SIZE decreased from 10k down to 1k to improve concurrent updates on maximum load --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0b012ba96c..9827a58a53 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -268,8 +268,8 @@ cassandra: sql: # Specify batch size for persisting attribute updates attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:1000}" + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:50}" stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_ATTRIBUTES_VALUE_NO_XSS_VALIDATION:false}" From c57f0bb8aa629cfcd260f1070db41d66c56bea12 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Oct 2023 10:41:43 +0200 Subject: [PATCH 15/36] SQL_TS_LATEST_BATCH_SIZE decreased from 10k down to 1k to improve concurrent updates on maximum load --- application/src/main/resources/thingsboard.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9827a58a53..4a3023d48f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -280,8 +280,8 @@ sql: batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:1000}" + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:50}" stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution update_by_latest_ts: "${SQL_TS_UPDATE_BY_LATEST_TIMESTAMP:true}" From 31f3c2a824a534c830aefc6d8122884859f5a2b6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 5 Oct 2023 11:39:11 +0200 Subject: [PATCH 16/36] CachedAttributesService find() many refactored in non-blocking manner --- .../dao/attributes/CachedAttributesService.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index faff81670b..9a1a1b7ee3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.stats.DefaultCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.cache.CacheExecutorService; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.sql.JpaExecutorService; import javax.annotation.PostConstruct; import java.util.ArrayList; @@ -61,6 +62,7 @@ public class CachedAttributesService implements AttributesService { public static final String LOCAL_CACHE_TYPE = "caffeine"; private final AttributesDao attributesDao; + private final JpaExecutorService jpaExecutorService; private final CacheExecutorService cacheExecutorService; private final DefaultCounter hitCounter; private final DefaultCounter missCounter; @@ -73,10 +75,12 @@ public class CachedAttributesService implements AttributesService { private boolean valueNoXssValidation; public CachedAttributesService(AttributesDao attributesDao, + JpaExecutorService jpaExecutorService, StatsFactory statsFactory, CacheExecutorService cacheExecutorService, TbTransactionalCache cache) { this.attributesDao = attributesDao; + this.jpaExecutorService = jpaExecutorService; this.cacheExecutorService = cacheExecutorService; this.cache = cache; @@ -134,12 +138,14 @@ public class CachedAttributesService implements AttributesService { } @Override - public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, Collection attributeKeys) { + public ListenableFuture> find(TenantId tenantId, EntityId entityId, String scope, final Collection attributeKeysNonUnique) { validate(entityId, scope); - attributeKeys = new LinkedHashSet<>(attributeKeys); // deduplicate the attributes + final var attributeKeys = new LinkedHashSet<>(attributeKeysNonUnique); // deduplicate the attributes attributeKeys.forEach(attributeKey -> Validator.validateString(attributeKey, "Incorrect attribute key " + attributeKey)); - Map> wrappedCachedAttributes = findCachedAttributes(entityId, scope, attributeKeys); + //CacheExecutor for Redis or DirectExecutor for local Caffeine + return Futures.transformAsync(cacheExecutor.submit(() -> findCachedAttributes(entityId, scope, attributeKeys)), + wrappedCachedAttributes -> { List cachedAttributes = wrappedCachedAttributes.values().stream() .map(TbCacheValueWrapper::get) @@ -155,7 +161,8 @@ public class CachedAttributesService implements AttributesService { List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - return cacheExecutor.submit(() -> { + // DB call should run in DB executor, not in cache-related executor + return jpaExecutorService.submit(() -> { var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); try { log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); @@ -179,6 +186,8 @@ public class CachedAttributesService implements AttributesService { throw e; } }); + + }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor } private Map> findCachedAttributes(EntityId entityId, String scope, Collection attributeKeys) { From 9788a87ca38db8d7233520dcdb87bfa5e1158936 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 4 Oct 2023 16:54:19 +0300 Subject: [PATCH 17/36] Fix notification requests deletion test; drop all notifications after test --- .../service/notification/AbstractNotificationApiTest.java | 4 ++++ .../server/service/notification/NotificationApiTest.java | 5 +++-- .../server/service/notification/NotificationRuleApiTest.java | 4 +--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java index 104b5afef4..27a0eed1a4 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/AbstractNotificationApiTest.java @@ -61,6 +61,7 @@ import org.thingsboard.server.dao.notification.NotificationRequestService; import org.thingsboard.server.dao.notification.NotificationRuleService; import org.thingsboard.server.dao.notification.NotificationTargetService; import org.thingsboard.server.dao.notification.NotificationTemplateService; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import java.net.URISyntaxException; import java.util.Arrays; @@ -90,6 +91,8 @@ public abstract class AbstractNotificationApiTest extends AbstractControllerTest protected NotificationTargetService notificationTargetService; @Autowired protected NotificationRequestService notificationRequestService; + @Autowired + protected SqlPartitioningRepository partitioningRepository; public static final String DEFAULT_NOTIFICATION_SUBJECT = "Just a test"; public static final NotificationType DEFAULT_NOTIFICATION_TYPE = NotificationType.GENERAL; @@ -100,6 +103,7 @@ public abstract class AbstractNotificationApiTest extends AbstractControllerTest notificationRuleService.deleteNotificationRulesByTenantId(TenantId.SYS_TENANT_ID); notificationTemplateService.deleteNotificationTemplatesByTenantId(TenantId.SYS_TENANT_ID); notificationTargetService.deleteNotificationTargetsByTenantId(TenantId.SYS_TENANT_ID); + partitioningRepository.dropPartitionsBefore("notification", Long.MAX_VALUE, 1); } protected NotificationTarget createNotificationTarget(UserId... usersIds) { diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 21d29833b9..26be4e9bd3 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -286,18 +286,19 @@ public class NotificationApiTest extends AbstractNotificationApiTest { @Test public void whenTenantIsDeleted_thenDeleteNotificationRequests() throws Exception { createDifferentTenant(); + TenantId tenantId = differentTenantId; NotificationTarget target = createNotificationTarget(savedDifferentTenantUser.getId()); int notificationsCount = 20; for (int i = 0; i < notificationsCount; i++) { NotificationRequest request = submitNotificationRequest(target.getId(), "Test " + i, NotificationDeliveryMethod.WEB); awaitNotificationRequest(request.getId()); } - List requests = notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(100)).getData(); + List requests = notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(100)).getData(); assertThat(requests).size().isEqualTo(notificationsCount); deleteDifferentTenant(); - assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(differentTenantId, EntityType.USER, new PageLink(1)).getTotalElements()) + assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements()) .isZero(); } diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index d08b1a121b..0a2854a28d 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -456,9 +456,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { loginSysAdmin(); notifications = await().atMost(30, TimeUnit.SECONDS) - .until(() -> getMyNotifications(true, 10).stream() - .filter(notification -> notification.getType() == NotificationType.RATE_LIMITS) - .collect(Collectors.toList()), list -> list.size() == 1); + .until(() -> getMyNotifications(true, 10), list -> list.size() == 1); assertThat(notifications).allSatisfy(notification -> { assertThat(notification.getSubject()).isEqualTo("Rate limits exceeded for tenant " + TEST_TENANT_NAME); }); From 261bec2b39b5dfbaac1fdf7809f5a8cd0ad6aed2 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Oct 2023 14:24:33 +0300 Subject: [PATCH 18/36] added missed yml parameter descriptions, script that check description exists for all parameters, added github action that run script --- .../check_yml_parameter_descriptions.yml | 24 + .../src/main/resources/thingsboard.yml | 641 +++++++++++++----- pull_request_template.md | 1 + tools/src/main/python/check_yml_file.py | 105 +++ 4 files changed, 608 insertions(+), 163 deletions(-) create mode 100644 .github/workflows/check_yml_parameter_descriptions.yml create mode 100644 tools/src/main/python/check_yml_file.py diff --git a/.github/workflows/check_yml_parameter_descriptions.yml b/.github/workflows/check_yml_parameter_descriptions.yml new file mode 100644 index 0000000000..240222d1b1 --- /dev/null +++ b/.github/workflows/check_yml_parameter_descriptions.yml @@ -0,0 +1,24 @@ +name: Check yml parameters have description +on: + pull_request: + paths: + - 'application/src/main/resources/thingsboard.yml' +jobs: + build: + name: check + runs-on: ubuntu-20.04 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10.2" + architecture: "x64" + env: + AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache + + - name: Run Verification Script + run: python3 tools/src/main/python/check_env_variables.py diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index d1e5113449..f04d2b8767 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -14,6 +14,7 @@ # limitations under the License. # +# Server common properties server: # Server bind address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -53,18 +54,27 @@ server: http2: # Enable/disable HTTP/2 support enabled: "${HTTP2_ENABLED:true}" + # Log errors with stacktrace when REST API throws exception with message "Please contact sysadmin" log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}" ws: + # Timeout for sending data to client WebSocket session in milliseconds send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}" # recommended timeout >= 30 seconds. Platform will attempt to send 'ping' request 3 times within the timeout ping_timeout: "${TB_SERVER_WS_PING_TIMEOUT:30000}" dynamic_page_link: + # Refresh rate of the dynamic alarm end entity data queries refresh_interval: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_INTERVAL_SEC:60}" + # Thread pool size to execute dynamic queries refresh_pool_size: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_POOL_SIZE:1}" + # Maximum number of dynamic queries per refresh interval. For example, no more than 10 alarm queries executed by user simultaneously in all browsers. max_alarm_queries_per_refresh_interval: "${TB_SERVER_WS_MAX_ALARM_QUERIES_PER_REFRESH_INTERVAL:10}" + # Maximum number of dynamic queries per user. For example, no more than 10 alarm widgets opened by user simultaneously in all browsers< max_per_user: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_MAX_PER_USER:10}" + # Maximum number of entities returned for single entity subscription. For example, no more than 10 000 entities on the map widget max_entities_per_data_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_DATA_SUBSCRIPTION:10000}" + # Maximum number of alarms returned for single alarm subscription. For example, no more than 10 000 alarms on the alarm widget max_entities_per_alarm_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_ALARM_SUBSCRIPTION:10000}" + # Maximum queue size of the websocket updates per session. This restriction prevents infinite updates of WS max_queue_messages_per_session: "${TB_SERVER_WS_DEFAULT_QUEUE_MESSAGES_PER_SESSION:1000}" rest: server_side_rpc: @@ -75,6 +85,7 @@ server: # Default value of the server side RPC timeout. default_timeout: "${DEFAULT_SERVER_SIDE_RPC_TIMEOUT:10000}" rate_limits: + # Limit that prohibits resetting the password for the user too often. The value of rate limit. By default, no more than 5 requests per hour reset_password_per_user: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" # Application info @@ -101,9 +112,12 @@ zk: # The delay is recommended because the initialization of rule chain actors is time-consuming. Avoiding unnecessary recalculations during a restart can enhance system performance and stability. recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" +# Cluster properties cluster: stats: + # Enable/Disable the cluster statistics. Calculates number of messages sent between cluster nodes based on each type enabled: "${TB_CLUSTER_STATS_ENABLED:false}" + # Interval of printing the cluster stats to the log file print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" # Plugins configuration parameters @@ -117,7 +131,7 @@ security: jwt: # Since 3.4.2 values are persisted to the database during install or upgrade. On Install, the key will be generated randomly if no custom value set. You can change it later from Web UI under SYS_ADMIN tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours) refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week). - tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" + tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" # User JWT Token issuer tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" # Base64 encoded # Enable/disable access to Tenant Administrators JWT token by System Administrator or Customer Users JWT token by Tenant Administrator user_token_access_enabled: "${SECURITY_USER_TOKEN_ACCESS_ENABLED:true}" @@ -129,28 +143,37 @@ security: # Time allowed to claim the device in milliseconds duration: "${SECURITY_CLAIM_DURATION:86400000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value basic: + # Enable/Disable basic security options enabled: "${SECURITY_BASIC_ENABLED:false}" oauth2: # Redirect URL where access code from external user management system will be processed loginProcessingUrl: "${SECURITY_OAUTH2_LOGIN_PROCESSING_URL:/login/oauth2/code/}" githubMapper: + # The email addresses that will be mapped from the URL emailUrl: "${SECURITY_OAUTH2_GITHUB_MAPPER_EMAIL_URL_KEY:https://api.github.com/user/emails}" java_cacerts: + # CA certificates keystore default path. Typically this keystore is at JAVA_HOME/lib/security/cacerts path: "${SECURITY_JAVA_CACERTS_PATH:${java.home}/lib/security/cacerts}" + # The password of the cacerts keystore file password: "${SECURITY_JAVA_CACERTS_PASSWORD:changeit}" mail: oauth2: - refreshTokenCheckingInterval: "${REFRESH_TOKEN_EXPIRATION_CHECKING_INTERVAL:86400}" # Number of seconds (1 day). + # Interval to check refresh token is going to expire in seconds(by default 1 day). + refreshTokenCheckingInterval: "${REFRESH_TOKEN_EXPIRATION_CHECKING_INTERVAL:86400}" # Usage statistics parameters usage: stats: report: + # Enable/Disable collection of statistics about API usage. Collected on a system and tenant level by default enabled: "${USAGE_STATS_REPORT_ENABLED:true}" + # Enable/Disable collection of statistics about API usage on a customer level enabled_per_customer: "${USAGE_STATS_REPORT_PER_CUSTOMER_ENABLED:false}" + # Interval of reporting the statistics. By default, the summarized statistics is sent every 10 seconds interval: "${USAGE_STATS_REPORT_INTERVAL:10}" check: + # Interval of checking the start of next cycle and re-enabling the blocked tenants/customers cycle: "${USAGE_STATS_CHECK_CYCLE:60000}" # In milliseconds. Default value is 3 minutes gauge_report_interval: "${USAGE_STATS_GAUGE_REPORT_INTERVAL:180000}" @@ -169,6 +192,7 @@ ui: # Base url for UI help assets base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-3.6}" +# Database telemetry parameters database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by single API call to fetch telemetry records ts: @@ -194,10 +218,12 @@ cassandra: hostname_validation: "${CASSANDRA_SSL_HOSTNAME_VALIDATION:true}" # Set trust store for client authentication of server (optional, uses trust store from default SSLContext if not set) trust_store: "${CASSANDRA_SSL_TRUST_STORE:}" + # The password for Cassandra trust store key trust_store_password: "${CASSANDRA_SSL_TRUST_STORE_PASSWORD:}" # Set key store for server authentication of client (optional, uses key store from default SSLContext if not set) # A key store is only needed if the Cassandra server requires client authentication key_store: "${CASSANDRA_SSL_KEY_STORE:}" + # The password for Cassandra key store key_store_password: "${CASSANDRA_SSL_KEY_STORE_PASSWORD:}" # Comma separated list of cipher suites (optional, uses Java default cipher suites if not set) cipher_suites: "${CASSANDRA_SSL_CIPHER_SUITES:}" @@ -211,7 +237,9 @@ cassandra: init_timeout_ms: "${CASSANDRA_CLUSTER_INIT_TIMEOUT_MS:300000}" # Specify cassandra claster initialization retry interval (if no hosts available during startup) init_retry_interval_ms: "${CASSANDRA_CLUSTER_INIT_RETRY_INTERVAL_MS:3000}" + # Cassandra max local requests per connection max_requests_per_connection_local: "${CASSANDRA_MAX_REQUESTS_PER_CONNECTION_LOCAL:32768}" + # Cassandra max remote requests per connection max_requests_per_connection_remote: "${CASSANDRA_MAX_REQUESTS_PER_CONNECTION_REMOTE:32768}" # Credential parameters credentials: "${CASSANDRA_USE_CREDENTIALS:false}" @@ -230,72 +258,93 @@ cassandra: # Cassandra cluster connection socket parameters # socket: + # Sets the timeout, in millisecond, of a native connection from ThingsBoard to Cassandra. The default value is 5000 connect_timeout: "${CASSANDRA_SOCKET_TIMEOUT:5000}" + # Timeout before closing the connection. Value set in milliseconds read_timeout: "${CASSANDRA_SOCKET_READ_TIMEOUT:20000}" + # Gets if TCP keep-alive must be used keep_alive: "${CASSANDRA_SOCKET_KEEP_ALIVE:true}" + # Enable/Disable reuse-address.The socket option allows for the reuse of local addresses and ports reuse_address: "${CASSANDRA_SOCKET_REUSE_ADDRESS:true}" + # Sets the linger-on-close timeout. By default, this option is not set by the driver. The actual value will be the default from the underlying Netty transport so_linger: "${CASSANDRA_SOCKET_SO_LINGER:}" + # Enable/Disable Nagle's algorithm tcp_no_delay: "${CASSANDRA_SOCKET_TCP_NO_DELAY:false}" + # Sets a hint to the size of the underlying buffers for incoming network I/O. By default, this option is not set by the driver. The actual value will be the default from the underlying Netty transport receive_buffer_size: "${CASSANDRA_SOCKET_RECEIVE_BUFFER_SIZE:}" + # Returns the hint to the size of the underlying buffers for outgoing network I/O. By default, this option is not set by the driver. The actual value will be the default from the underlying Netty transport send_buffer_size: "${CASSANDRA_SOCKET_SEND_BUFFER_SIZE:}" # Cassandra cluster connection query parameters # query: + # Consistency levels in Cassandra can be configured to manage availability versus data accuracy. The consistency level defaults to ONE for all write and read operations read_consistency_level: "${CASSANDRA_READ_CONSISTENCY_LEVEL:ONE}" + # Consistency levels in Cassandra can be configured to manage availability versus data accuracy. The consistency level defaults to ONE for all write and read operations write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}" + # The fetch size specifies how many rows will be returned at once by Cassandra (in other words, it’s the size of each page) default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}" # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" + # Enable/Disable timestamp key-value partioning on read queries use_ts_key_value_partitioning_on_read: "${USE_TS_KV_PARTITIONING_ON_READ:true}" + # The number of partitions that are cached in memory of each service. Useful to decrease load of re-inserting same partitions again ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}" + # Timeseries Time To Live (in seconds) for Cassandra Record. 0 - record is never expired ts_key_value_ttl: "${TS_KV_TTL:0}" + # Maximum number of Cassandra queries that are waiting for execution buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}" + # Maximum number of concurrent Cassandra queries concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}" + # Max time in milliseconds query waits for execution permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}" + # Amount of threads to dispatch cassandra queries dispatcher_threads: "${CASSANDRA_QUERY_DISPATCHER_THREADS:2}" callback_threads: "${CASSANDRA_QUERY_CALLBACK_THREADS:4}" # Buffered rate executor (read, write), for managing I/O rate. See "nosql-*-callback" threads in JMX result_processing_threads: "${CASSANDRA_QUERY_RESULT_PROCESSING_THREADS:50}" # Result set transformer and processing. See "cassandra-callback" threads in JMX + # Cassandra query queue polling interval in milliseconds poll_ms: "${CASSANDRA_QUERY_POLL_MS:50}" + # Interval in milliseconds for printing Cassandra query queue statistic rate_limit_print_interval_ms: "${CASSANDRA_QUERY_RATE_LIMIT_PRINT_MS:10000}" # set all data types values except target to null for the same ts on save set_null_values_enabled: "${CASSANDRA_QUERY_SET_NULL_VALUES_ENABLED:false}" # log one of cassandra queries with specified frequency (0 - logging is disabled) print_queries_freq: "${CASSANDRA_QUERY_PRINT_FREQ:0}" tenant_rate_limits: + # Whether to print rate-limited tenant names when printing Cassandra query queue statistic print_tenant_names: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_PRINT_TENANT_NAMES:false}" # SQL configuration parameters sql: # Specify batch size for persisting attribute updates attributes: - batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" + batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" # Batch size for persisting attribute updates + batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" # Max timeout for attributes entries queue polling. Value set in milliseconds + stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing attributes updates statistic batch_threads: "${SQL_ATTRIBUTES_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution - value_no_xss_validation: "${SQL_ATTRIBUTES_VALUE_NO_XSS_VALIDATION:false}" + value_no_xss_validation: "${SQL_ATTRIBUTES_VALUE_NO_XSS_VALIDATION:false}" # If true attribute values will be checked for XSS vulnerability ts: - batch_size: "${SQL_TS_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" + batch_size: "${SQL_TS_BATCH_SIZE:10000}" # Batch size for persisting timeseries inserts + batch_max_delay: "${SQL_TS_BATCH_MAX_DELAY_MS:100}" # Max timeout for time-series entries queue polling. Value set in milliseconds + stats_print_interval_ms: "${SQL_TS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing timeseries insert statistic batch_threads: "${SQL_TS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution - value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" + value_no_xss_validation: "${SQL_TS_VALUE_NO_XSS_VALIDATION:false}" # If true telemetry values will be checked for XSS vulnerability ts_latest: - batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" + batch_size: "${SQL_TS_LATEST_BATCH_SIZE:10000}" # Batch size for persisting latest telemetry updates + batch_max_delay: "${SQL_TS_LATEST_BATCH_MAX_DELAY_MS:100}" # Maximum timeout for latest telemetry entries queue polling. The value set in milliseconds + stats_print_interval_ms: "${SQL_TS_LATEST_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing latest telemetry updates statistic batch_threads: "${SQL_TS_LATEST_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution - update_by_latest_ts: "${SQL_TS_UPDATE_BY_LATEST_TIMESTAMP:true}" + update_by_latest_ts: "${SQL_TS_UPDATE_BY_LATEST_TIMESTAMP:true}" # Update latest values only if the timestamp of the new record is greater or equals than timestamp of the previously saved latest value. Latest values are stored separately from historical values for fast lookup from DB. Insert of historical value happens in any case events: - batch_size: "${SQL_EVENTS_BATCH_SIZE:10000}" - batch_max_delay: "${SQL_EVENTS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_EVENTS_BATCH_STATS_PRINT_MS:10000}" + batch_size: "${SQL_EVENTS_BATCH_SIZE:10000}" # Batch size for persisting latest telemetry updates + batch_max_delay: "${SQL_EVENTS_BATCH_MAX_DELAY_MS:100}" # Max timeout for latest telemetry entries queue polling. The value set in milliseconds + stats_print_interval_ms: "${SQL_EVENTS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing latest telemetry updates statistic batch_threads: "${SQL_EVENTS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week. debug_partition_size: "${SQL_EVENTS_DEBUG_PARTITION_SIZE_HOURS:1}" # Number of hours to partition the debug events. The current value corresponds to one hour. edge_events: - batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" - batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_EDGE_EVENTS_BATCH_STATS_PRINT_MS:10000}" + batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" # Batch size for persisting latest telemetry updates + batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" # Max timeout for latest telemetry entries queue polling. The value set in milliseconds + stats_print_interval_ms: "${SQL_EDGE_EVENTS_BATCH_STATS_PRINT_MS:10000}" # Interval in milliseconds for printing latest telemetry updates statistic partition_size: "${SQL_EDGE_EVENTS_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week. audit_logs: partition_size: "${SQL_AUDIT_LOGS_PARTITION_SIZE_HOURS:168}" # Default value - 1 week @@ -309,8 +358,11 @@ sql: remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" # Specify whether to log database queries and their parameters generated by entity query repository log_queries: "${SQL_LOG_QUERIES:false}" + # Threshold of slow SQL queries to log. The value set in milliseconds log_queries_threshold: "${SQL_LOG_QUERIES_THRESHOLD:5000}" + # Enable/Disable logging statistic information about tenants log_tenant_stats: "${SQL_LOG_TENANT_STATS:true}" + # Interval in milliseconds for printing latest statistic information about tenant log_tenant_stats_interval_ms: "${SQL_LOG_TENANT_STATS_INTERVAL_MS:60000}" postgres: # Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE. @@ -321,10 +373,12 @@ sql: batch_threads: "${SQL_TIMESCALE_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution ttl: ts: + # The parameter to specify whether to use TTL (Time To Live) for timeseries records enabled: "${SQL_TTL_TS_ENABLED:true}" execution_interval_ms: "${SQL_TTL_TS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: + # The parameter to specify whether to use TTL (Time To Live) for event records enabled: "${SQL_TTL_EVENTS_ENABLED:true}" execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:3600000}" # Number of milliseconds (max random initial delay and fixed period). # Number of seconds. TTL is disabled by default. Accuracy of the cleanup depends on the sql.events.partition_size parameter. @@ -332,21 +386,21 @@ sql: # Number of seconds. The current value corresponds to one week. Accuracy of the cleanup depends on the sql.events.debug_partition_size parameter. debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" edge_events: - enabled: "${SQL_TTL_EDGE_EVENTS_ENABLED:true}" + enabled: "${SQL_TTL_EDGE_EVENTS_ENABLED:true}" # The parameter to specify whether to use TTL (Time To Live) for egde event records execution_interval_ms: "${SQL_TTL_EDGE_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day edge_events_ttl: "${SQL_TTL_EDGE_EVENTS_TTL:2628000}" # Number of seconds. The current value corresponds to one month alarms: checking_interval: "${SQL_ALARMS_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # To delete outdated alarms not all at once but in batches rpc: - enabled: "${SQL_TTL_RPC_ENABLED:true}" + enabled: "${SQL_TTL_RPC_ENABLED:true}" # The parameter to specify whether to use TTL (Time To Live) for rpc call records checking_interval: "${SQL_RPC_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours audit_logs: - enabled: "${SQL_TTL_AUDIT_LOGS_ENABLED:true}" + enabled: "${SQL_TTL_AUDIT_LOGS_ENABLED:true}" # The parameter to specify whether to use TTL (Time To Live) for audit log records ttl: "${SQL_TTL_AUDIT_LOGS_SECS:0}" # Disabled by default. Accuracy of the cleanup depends on the sql.audit_logs.partition_size checking_interval_ms: "${SQL_TTL_AUDIT_LOGS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day notifications: - enabled: "${SQL_TTL_NOTIFICATIONS_ENABLED:true}" + enabled: "${SQL_TTL_NOTIFICATIONS_ENABLED:true}" # The parameter to specify whether to use TTL (Time To Live) for notification center records ttl: "${SQL_TTL_NOTIFICATIONS_SECS:2592000}" # Default value - 30 days checking_interval_ms: "${SQL_TTL_NOTIFICATIONS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day relations: @@ -357,17 +411,17 @@ sql: # Actor system parameters actors: system: - throughput: "${ACTORS_SYSTEM_THROUGHPUT:5}" - scheduler_pool_size: "${ACTORS_SYSTEM_SCHEDULER_POOL_SIZE:1}" - max_actor_init_attempts: "${ACTORS_SYSTEM_MAX_ACTOR_INIT_ATTEMPTS:10}" - app_dispatcher_pool_size: "${ACTORS_SYSTEM_APP_DISPATCHER_POOL_SIZE:1}" - tenant_dispatcher_pool_size: "${ACTORS_SYSTEM_TENANT_DISPATCHER_POOL_SIZE:2}" - device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" - rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:4}" + throughput: "${ACTORS_SYSTEM_THROUGHPUT:5}" # Number of messages the actor system will process per actor before switching to processing of messages for next actor + scheduler_pool_size: "${ACTORS_SYSTEM_SCHEDULER_POOL_SIZE:1}" # Thread pool size for actor system scheduler + max_actor_init_attempts: "${ACTORS_SYSTEM_MAX_ACTOR_INIT_ATTEMPTS:10}" # Maximum number of attempts to init the actor before disabling the actor + app_dispatcher_pool_size: "${ACTORS_SYSTEM_APP_DISPATCHER_POOL_SIZE:1}" # Thread pool size for main actor system dispatcher + tenant_dispatcher_pool_size: "${ACTORS_SYSTEM_TENANT_DISPATCHER_POOL_SIZE:2}" # Thread pool size for actor system dispatcher that process messages for tenant actors + device_dispatcher_pool_size: "${ACTORS_SYSTEM_DEVICE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for device actors + rule_dispatcher_pool_size: "${ACTORS_SYSTEM_RULE_DISPATCHER_POOL_SIZE:4}" # Thread pool size for actor system dispatcher that process messages for rule engine (chain/node) actors tenant: - create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" + create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" # Create components in initialization session: - max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:1}" + max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:1}" # Max number of concurrent sessions per device sync: # Default timeout for processing request using synchronous session (HTTP, CoAP) in milliseconds timeout: "${ACTORS_SESSION_SYNC_TIMEOUT:10000}" @@ -390,7 +444,9 @@ actors: # Errors for particular actor are persisted once per specified amount of milliseconds error_persist_frequency: "${ACTORS_RULE_CHAIN_ERROR_FREQUENCY:3000}" debug_mode_rate_limits_per_tenant: + # Enable/Disable the rate limit of persisted debug events for all rule nodes per tenant enabled: "${ACTORS_RULE_CHAIN_DEBUG_MODE_RATE_LIMITS_PER_TENANT_ENABLED:true}" + # The value of DEBUG mode rate limit. By default, no more then 50 thousand events per hour configuration: "${ACTORS_RULE_CHAIN_DEBUG_MODE_RATE_LIMITS_PER_TENANT_CONFIGURATION:50000:3600}" node: # Errors for particular actor are persisted once per specified amount of milliseconds @@ -414,9 +470,12 @@ actors: statistics: # Enable/disable actor statistics enabled: "${ACTORS_STATISTICS_ENABLED:true}" + # Frequency of printing the JS executor statistics js_print_interval_ms: "${ACTORS_JS_STATISTICS_PRINT_INTERVAL_MS:10000}" + # Actors statistic persistence frequency in milliseconds persist_frequency: "${ACTORS_STATISTICS_PERSIST_FREQUENCY:3600000}" +# Cache parameters cache: # caffeine or redis type: "${CACHE_TYPE:caffeine}" @@ -426,113 +485,118 @@ cache: enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" specs: relations: - timeToLiveInMinutes: "${CACHE_SPECS_RELATIONS_TTL:1440}" - maxSize: "${CACHE_SPECS_RELATIONS_MAX_SIZE:10000}" # maxSize: 0 means the cache is disabled + timeToLiveInMinutes: "${CACHE_SPECS_RELATIONS_TTL:1440}" # Relations cache max size + maxSize: "${CACHE_SPECS_RELATIONS_MAX_SIZE:10000}" # 0 means the cache is disabled deviceCredentials: - timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_CREDENTIALS_TTL:1440}" - maxSize: "${CACHE_SPECS_DEVICE_CREDENTIALS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_CREDENTIALS_TTL:1440}" # Device credentials cache TTL + maxSize: "${CACHE_SPECS_DEVICE_CREDENTIALS_MAX_SIZE:10000}" # 0 means the cache is disabled devices: - timeToLiveInMinutes: "${CACHE_SPECS_DEVICES_TTL:1440}" - maxSize: "${CACHE_SPECS_DEVICES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_DEVICES_TTL:1440}" # Device cache TTL + maxSize: "${CACHE_SPECS_DEVICES_MAX_SIZE:10000}" # 0 means the cache is disabled sessions: - timeToLiveInMinutes: "${CACHE_SPECS_SESSIONS_TTL:1440}" - maxSize: "${CACHE_SPECS_SESSIONS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_SESSIONS_TTL:1440}" # Sessions cache TTL + maxSize: "${CACHE_SPECS_SESSIONS_MAX_SIZE:10000}" # 0 means the cache is disabled assets: - timeToLiveInMinutes: "${CACHE_SPECS_ASSETS_TTL:1440}" - maxSize: "${CACHE_SPECS_ASSETS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_ASSETS_TTL:1440}" # Asset cache TTL + maxSize: "${CACHE_SPECS_ASSETS_MAX_SIZE:10000}" # 0 means the cache is disabled entityViews: - timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_VIEWS_TTL:1440}" - maxSize: "${CACHE_SPECS_ENTITY_VIEWS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_VIEWS_TTL:1440}" # Entity view cache TTL + maxSize: "${CACHE_SPECS_ENTITY_VIEWS_MAX_SIZE:10000}" # 0 means the cache is disabled claimDevices: - timeToLiveInMinutes: "${CACHE_SPECS_CLAIM_DEVICES_TTL:1440}" - maxSize: "${CACHE_SPECS_CLAIM_DEVICES_MAX_SIZE:1000}" + timeToLiveInMinutes: "${CACHE_SPECS_CLAIM_DEVICES_TTL:1440}" # Claim devices cache TTL + maxSize: "${CACHE_SPECS_CLAIM_DEVICES_MAX_SIZE:1000}" # 0 means the cache is disabled securitySettings: - timeToLiveInMinutes: "${CACHE_SPECS_SECURITY_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_SECURITY_SETTINGS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_SECURITY_SETTINGS_TTL:1440}" # Security settings cache TTL + maxSize: "${CACHE_SPECS_SECURITY_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled tenantProfiles: - timeToLiveInMinutes: "${CACHE_SPECS_TENANT_PROFILES_TTL:1440}" - maxSize: "${CACHE_SPECS_TENANT_PROFILES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_TENANT_PROFILES_TTL:1440}" # Tenant profiles cache TTL + maxSize: "${CACHE_SPECS_TENANT_PROFILES_MAX_SIZE:10000}" # 0 means the cache is disabled tenants: - timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" - maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" # Tenant cache TTL + maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" # 0 means the cache is disabled tenantsExist: # environment variables are intentionally the same as in 'tenants' cache to be equal. timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" - maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" + maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" # 0 means the cache is disabled deviceProfiles: - timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_PROFILES_TTL:1440}" - maxSize: "${CACHE_SPECS_DEVICE_PROFILES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_PROFILES_TTL:1440}" # Device profile cache TTL + maxSize: "${CACHE_SPECS_DEVICE_PROFILES_MAX_SIZE:10000}" # 0 means the cache is disabled assetProfiles: - timeToLiveInMinutes: "${CACHE_SPECS_ASSET_PROFILES_TTL:1440}" - maxSize: "${CACHE_SPECS_ASSET_PROFILES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_ASSET_PROFILES_TTL:1440}" # Asset profile cache TTL + maxSize: "${CACHE_SPECS_ASSET_PROFILES_MAX_SIZE:10000}" # 0 means the cache is disabled notificationSettings: - timeToLiveInMinutes: "${CACHE_SPECS_NOTIFICATION_SETTINGS_TTL:10}" - maxSize: "${CACHE_SPECS_NOTIFICATION_SETTINGS_MAX_SIZE:1000}" + timeToLiveInMinutes: "${CACHE_SPECS_NOTIFICATION_SETTINGS_TTL:10}" # Noification settings cache TTL + maxSize: "${CACHE_SPECS_NOTIFICATION_SETTINGS_MAX_SIZE:1000}" # 0 means the cache is disabled sentNotifications: - timeToLiveInMinutes: "${CACHE_SPECS_SENT_NOTIFICATIONS_TTL:1440}" - maxSize: "${CACHE_SPECS_SENT_NOTIFICATIONS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_SENT_NOTIFICATIONS_TTL:1440}" # Sent notifications cache TTL + maxSize: "${CACHE_SPECS_SENT_NOTIFICATIONS_MAX_SIZE:10000}" # 0 means the cache is disabled attributes: - timeToLiveInMinutes: "${CACHE_SPECS_ATTRIBUTES_TTL:1440}" - maxSize: "${CACHE_SPECS_ATTRIBUTES_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_ATTRIBUTES_TTL:1440}" # Attributes cache TTL + maxSize: "${CACHE_SPECS_ATTRIBUTES_MAX_SIZE:100000}" # 0 means the cache is disabled userSessionsInvalidation: # The value of this TTL is ignored and replaced by JWT refresh token expiration time timeToLiveInMinutes: "0" - maxSize: "${CACHE_SPECS_USERS_UPDATE_TIME_MAX_SIZE:10000}" + maxSize: "${CACHE_SPECS_USERS_UPDATE_TIME_MAX_SIZE:10000}" # 0 means the cache is disabled otaPackages: - timeToLiveInMinutes: "${CACHE_SPECS_OTA_PACKAGES_TTL:60}" - maxSize: "${CACHE_SPECS_OTA_PACKAGES_MAX_SIZE:10}" + timeToLiveInMinutes: "${CACHE_SPECS_OTA_PACKAGES_TTL:60}" # Ota packages cache TTL + maxSize: "${CACHE_SPECS_OTA_PACKAGES_MAX_SIZE:10}" # 0 means the cache is disabled otaPackagesData: - timeToLiveInMinutes: "${CACHE_SPECS_OTA_PACKAGES_DATA_TTL:60}" - maxSize: "${CACHE_SPECS_OTA_PACKAGES_DATA_MAX_SIZE:10}" + timeToLiveInMinutes: "${CACHE_SPECS_OTA_PACKAGES_DATA_TTL:60}" # Ota packages data cache TTL + maxSize: "${CACHE_SPECS_OTA_PACKAGES_DATA_MAX_SIZE:10}" # 0 means the cache is disabled edges: - timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" - maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" # Edges cache TTL + maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" # 0 means the cache is disabled repositorySettings: - timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" # Repository settings cache TTL + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled autoCommitSettings: - timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" # Autocommit settings cache TTL + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" # 0 means the cache is disabled twoFaVerificationCodes: - timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}" - maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}" # Two fa carification codes cache TTL + maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}" # 0 means the cache is disabled versionControlTask: - timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:20}" - maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:20}" # Version control task cache TTL + maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" # 0 means the cache is disabled userSettings: - timeToLiveInMinutes: "${CACHE_SPECS_USER_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_USER_SETTINGS_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_USER_SETTINGS_TTL:1440}" # User settings cache TTL + maxSize: "${CACHE_SPECS_USER_SETTINGS_MAX_SIZE:100000}" # 0 means the cache is disabled dashboardTitles: - timeToLiveInMinutes: "${CACHE_SPECS_DASHBOARD_TITLES_TTL:1440}" - maxSize: "${CACHE_SPECS_DASHBOARD_TITLES_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_DASHBOARD_TITLES_TTL:1440}" # Dashboard titles cache TTL + maxSize: "${CACHE_SPECS_DASHBOARD_TITLES_MAX_SIZE:100000}" # 0 means the cache is disabled entityCount: - timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_COUNT_TTL:1440}" - maxSize: "${CACHE_SPECS_ENTITY_COUNT_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_ENTITY_COUNT_TTL:1440}" # Entity count cache TTL + maxSize: "${CACHE_SPECS_ENTITY_COUNT_MAX_SIZE:100000}" # 0 means the cache is disabled resourceInfo: - timeToLiveInMinutes: "${CACHE_SPECS_RESOURCE_INFO_TTL:1440}" - maxSize: "${CACHE_SPECS_RESOURCE_INFO_MAX_SIZE:100000}" + timeToLiveInMinutes: "${CACHE_SPECS_RESOURCE_INFO_TTL:1440}" # Resource info cache TTL + maxSize: "${CACHE_SPECS_RESOURCE_INFO_MAX_SIZE:100000}" # 0 means the cache is disabled alarmTypes: - timeToLiveInMinutes: "${CACHE_SPECS_ALARM_TYPES_TTL:60}" - maxSize: "${CACHE_SPECS_ALARM_TYPES_MAX_SIZE:10000}" + timeToLiveInMinutes: "${CACHE_SPECS_ALARM_TYPES_TTL:60}" # Alarm types cache TTL + maxSize: "${CACHE_SPECS_ALARM_TYPES_MAX_SIZE:10000}" # 0 means the cache is disabled # deliberately placed outside 'specs' group above notificationRules: - timeToLiveInMinutes: "${CACHE_SPECS_NOTIFICATION_RULES_TTL:30}" - maxSize: "${CACHE_SPECS_NOTIFICATION_RULES_MAX_SIZE:1000}" + timeToLiveInMinutes: "${CACHE_SPECS_NOTIFICATION_RULES_TTL:30}" # notification rules cache TTL + maxSize: "${CACHE_SPECS_NOTIFICATION_RULES_MAX_SIZE:1000}" # 0 means the cache is disabled rateLimits: - timeToLiveInMinutes: "${CACHE_SPECS_RATE_LIMITS_TTL:120}" - maxSize: "${CACHE_SPECS_RATE_LIMITS_MAX_SIZE:200000}" + timeToLiveInMinutes: "${CACHE_SPECS_RATE_LIMITS_TTL:120}" # Rate limits cache TTL + maxSize: "${CACHE_SPECS_RATE_LIMITS_MAX_SIZE:200000}" # 0 means the cache is disabled -#Disable this because it is not required. -spring.data.redis.repositories.enabled: false +# Spring data properties +spring.data.redis.repositories.enabled: false # Disable this because it is not required. +# Redis configuration properties redis: # standalone or cluster or sentinel connection: + # Redis deployment type: Standalone (single Redis node deployment) OR Cluster type: "${REDIS_CONNECTION_TYPE:standalone}" standalone: + # Redis connection host host: "${REDIS_HOST:localhost}" + # Redis connection port port: "${REDIS_PORT:6379}" + # Use default Redis configuration file useDefaultClientConfig: "${REDIS_USE_DEFAULT_CLIENT_CONFIG:true}" # this value may be used only if you used not default ClientConfig clientName: "${REDIS_CLIENT_NAME:standalone}" @@ -564,31 +628,37 @@ redis: password: "${REDIS_PASSWORD:}" # pool config pool_config: + # Maximum number of connections that can be allocated by the connection pool maxTotal: "${REDIS_POOL_CONFIG_MAX_TOTAL:128}" + # Maximum number of idle connections that can be maintained in the pool without being closed maxIdle: "${REDIS_POOL_CONFIG_MAX_IDLE:128}" + # Minumum number of idle connections that can be maintained in the pool without being closed minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" + # Enable/Disable PING command send when a connection is borrowed testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + # The property is used to specify whether to test the connection before returning it to the connection pool. testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + # The property is used in the context of connection pooling in Redis testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" + # Minimum amount of time that an idle connection should be idle before it can be evicted from the connection pool. Value set in milliseconds minEvictableMs: "${REDIS_POOL_CONFIG_MIN_EVICTABLE_MS:60000}" + # Specifies the time interval in milliseconds between two consecutive eviction runs evictionRunsMs: "${REDIS_POOL_CONFIG_EVICTION_RUNS_MS:30000}" + # Maximum time in milliseconds where a client is willing to wait for a connection from the pool when all connections are exhausted maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}" + # Specifies the number of connections to test for eviction during each eviction run numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}" + # Determines the behavior when a thread requests a connection from the pool but there are no available connections and the pool cannot create more due to the maxTotal configuration blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # TTL for short-living SET commands that are used to replace DEL in order to enable transaction support evictTtlInMs: "${REDIS_EVICT_TTL_MS:60000}" -# Check new version updates parameters +# Update version properties updates: - # Enable/disable updates checking. + # Enable/disable checks for new version enabled: "${UPDATES_ENABLED:true}" -spring.main.allow-circular-references: "true" - -# spring freemarker configuration -spring.freemarker.checkTemplateLocation: "false" - -# spring CORS configuration +# Spring CORS configuration spring.mvc.cors: mappings: # Intercept path @@ -604,45 +674,49 @@ spring.mvc.cors: #Set whether credentials are supported. When not set, credentials are not supported. allow-credentials: "true" -# The default timeout for asynchronous requests in milliseconds -spring.mvc.async.request-timeout: "${SPRING_MVC_ASYNC_REQUEST_TIMEOUT:30000}" - -# For endpoints matching in Swagger -spring.mvc.pathmatch.matching-strategy: "ANT_PATH_MATCHER" - -# spring serve gzip compressed static resources +# General spring properties +spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed or not. +spring.freemarker.checkTemplateLocation: "false" # spring freemarker configuration +spring.mvc.async.request-timeout: "${SPRING_MVC_ASYNC_REQUEST_TIMEOUT:30000}" # The default timeout for asynchronous requests in milliseconds +spring.mvc.pathmatch.matching-strategy: "ANT_PATH_MATCHER" # For endpoints matching in Swagger spring.resources.chain: - compressed: "true" + compressed: "true" # This property enables or disables support for serving pre-compressed resources (for example, a .gzip or .br file) strategy: content: - enabled: "true" + enabled: "true" # This property enables or disables the content Version Strategy. This strategy allows Spring to generate a unique version for static resources, which is based on the content of the resource -spring.servlet.multipart.max-file-size: "50MB" -spring.servlet.multipart.max-request-size: "50MB" +spring.servlet.multipart.max-file-size: "50MB" # Total file size cannot exceed 50MB when configuring file uploads +spring.servlet.multipart.max-request-size: "50MB" # Total request size for a multipart/form-data cannot exceed 50MB -spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" -# Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used -spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" +spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" #Fix Postgres JPA Error (Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented) +spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" # Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used # SQL DAO Configuration spring: data: jpa: repositories: - enabled: "true" + enabled: "true" # Enable/Disable the Spring Data JPA repositories support jpa: properties: - javax.persistence.query.timeout: "${JAVAX_PERSISTENCE_QUERY_TIMEOUT:30000}" - open-in-view: "false" + javax.persistence.query.timeout: "${JAVAX_PERSISTENCE_QUERY_TIMEOUT:30000}" # General timeout for JDBC queries + open-in-view: "false" # Enabled by default. Therefore, database queries may be performed during view rendering. Explicitly configure spring.jpa.open-in-view to disable this warning hibernate: + # You can set a Hibernate feature that controls the DDL behavior in a more fine-grained way. The standard Hibernate property values are none, validate, update, create-drop. Spring Boot chooses a default value for you based on whether it thinks your database is embedded (default create-drop) or not (default none) ddl-auto: "none" datasource: + # Database driver for Spring JPA - org.postgresql.Driver driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" + # Database connection URL url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" + # Database user name username: "${SPRING_DATASOURCE_USERNAME:postgres}" + # Database user password password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: + # This property controls the amount of time that a connection can be out of the pool before a message is logged indicating a possible connection leak. A value of 0 means leak detection is disabled leakDetectionThreshold: "${SPRING_DATASOURCE_HIKARI_LEAK_DETECTION_THRESHOLD:0}" + # This property allows the number of connections in the pool to increase as demand increases. At the same time, the property ensures that the pool doesn't grow to the point of exhausting a system's resources, which ultimately affects an application's performance and availability maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" registerMbeans: "${SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX @@ -654,21 +728,21 @@ audit-log: # Allowed values: OFF (disable), W (log write operations), RW (log read and write operations) logging-level: mask: - "device": "${AUDIT_LOG_MASK_DEVICE:W}" - "asset": "${AUDIT_LOG_MASK_ASSET:W}" - "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}" - "widget_type": "${AUDIT_LOG_MASK_WIDGET_TYPE:W}" - "widgets_bundle": "${AUDIT_LOG_MASK_WIDGETS_BUNDLE:W}" - "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}" - "user": "${AUDIT_LOG_MASK_USER:W}" - "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}" - "alarm": "${AUDIT_LOG_MASK_ALARM:W}" - "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}" - "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}" - "asset_profile": "${AUDIT_LOG_MASK_ASSET_PROFILE:W}" - "edge": "${AUDIT_LOG_MASK_EDGE:W}" - "tb_resource": "${AUDIT_LOG_MASK_RESOURCE:W}" - "ota_package": "${AUDIT_LOG_MASK_OTA_PACKAGE:W}" + "device": "${AUDIT_LOG_MASK_DEVICE:W}" # Device logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "asset": "${AUDIT_LOG_MASK_ASSET:W}" # Asset logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}" # Dashboard logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "widget_type": "${AUDIT_LOG_MASK_WIDGET_TYPE:W}" # Widget type logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "widgets_bundle": "${AUDIT_LOG_MASK_WIDGETS_BUNDLE:W}" # Widget bundles logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}" # Customer logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "user": "${AUDIT_LOG_MASK_USER:W}" # User logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}" # Rule chain logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "alarm": "${AUDIT_LOG_MASK_ALARM:W}" # Alarm logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}" # Entity view logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}" # Device profile logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "asset_profile": "${AUDIT_LOG_MASK_ASSET_PROFILE:W}" # Asset profile logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "edge": "${AUDIT_LOG_MASK_EDGE:W}" # Edge logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "tb_resource": "${AUDIT_LOG_MASK_RESOURCE:W}" # TB resource logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation + "ota_package": "${AUDIT_LOG_MASK_OTA_PACKAGE:W}" # Ota package logging levels. Allowed values: OFF (disable), W (log write operations), RW (log read and write operation sink: # Type of external sink. possible options: none, elasticsearch type: "${AUDIT_LOG_SINK_TYPE:none}" @@ -681,25 +755,31 @@ audit-log: # https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.dd}" scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https - host: "${AUDIT_LOG_SINK_HOST:localhost}" - port: "${AUDIT_LOG_SINK_PORT:9200}" - user_name: "${AUDIT_LOG_SINK_USER_NAME:}" - password: "${AUDIT_LOG_SINK_PASSWORD:}" + host: "${AUDIT_LOG_SINK_HOST:localhost}" # Host of external sink system + port: "${AUDIT_LOG_SINK_PORT:9200}" # Port of external sink system + user_name: "${AUDIT_LOG_SINK_USER_NAME:}" # Username used to access external sink system + password: "${AUDIT_LOG_SINK_PASSWORD:}" # Password used to access external sink system +# State properties state: - # Should be greater then transport.sessions.report_timeout + # Should be greater than transport.sessions.report_timeout defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" # Interval for checking the device state after a specified period of time. Time in seconds # Controls whether we store device 'active' flag in attributes (default) or telemetry. # If you device to change this parameter, you should re-create the device info view as one of the following: # If 'persistToTelemetry' is changed from 'false' to 'true': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_ts_view;' # If 'persistToTelemetry' is changed from 'true' to 'false': 'CREATE OR REPLACE VIEW device_info_view AS SELECT * FROM device_info_active_attribute_view;' persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" +# Tbel properties tbel: + # Enable/Disable TBEL feature. enabled: "${TBEL_ENABLED:true}" + # Limit on the number of arguments that are passed to the function to execute the script max_total_args_size: "${TBEL_MAX_TOTAL_ARGS_SIZE:100000}" + # Maximum allowed symbols in a result after processing a script max_result_size: "${TBEL_MAX_RESULT_SIZE:300000}" + # Maximum allowed symbols in script body max_script_body_size: "${TBEL_MAX_SCRIPT_BODY_SIZE:50000}" # Maximum allowed TBEL script execution memory max_memory_limit_mb: "${TBEL_MAX_MEMORY_LIMIT_MB: 8}" @@ -711,15 +791,23 @@ tbel: max_black_list_duration_sec: "${TBEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${TBEL_THREAD_POOL_SIZE:50}" + # Maximum cache size of TBEL compiled scripts compiled_scripts_cache_size: "${TBEL_COMPILED_SCRIPTS_CACHE_SIZE:1000}" stats: + # Enable/Disable stats collection for TBEL engine enabled: "${TB_TBEL_STATS_ENABLED:false}" + # Interval of logging for TBEL stats print_interval_ms: "${TB_TBEL_STATS_PRINT_INTERVAL_MS:10000}" +# JS properties js: - evaluator: "${JS_EVALUATOR:local}" # local/remote + # local (Nashorn Engine, deprecated) OR remote JS-Executors (NodeJS) + evaluator: "${JS_EVALUATOR:local}" + # Limit on the number of arguments that are passed to the function to execute the script max_total_args_size: "${JS_MAX_TOTAL_ARGS_SIZE:100000}" + # Maximum allowed symbols in a result after processing a script max_result_size: "${JS_MAX_RESULT_SIZE:300000}" + # Maximum allowed symbols in script body max_script_body_size: "${JS_MAX_SCRIPT_BODY_SIZE:50000}" # Built-in JVM JavaScript environment properties local: @@ -738,7 +826,9 @@ js: # Maximum time in seconds for black listed function to stay in the list. max_black_list_duration_sec: "${LOCAL_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: + # Enable/Disable stats collection for local JS executor enabled: "${TB_JS_LOCAL_STATS_ENABLED:false}" + # Interval of logging for local JS executor stats print_interval_ms: "${TB_JS_LOCAL_STATS_PRINT_INTERVAL_MS:10000}" # Remote JavaScript environment properties remote: @@ -749,12 +839,17 @@ js: # Maximum time in seconds for black listed function to stay in the list. max_black_list_duration_sec: "${REMOTE_JS_SANDBOX_MAX_BLACKLIST_DURATION_SEC:60}" stats: + # Enable/Disable stats collection for remote JS executor enabled: "${TB_JS_REMOTE_STATS_ENABLED:false}" + # Interval of logging for remote JS executor stats print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" +# Transport configuration properties transport: sessions: + # Inactivity timeout for device session in transport service. The last activity time of the device session is updated if device sends any message, including keepalive messages inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" + # Interval of periodic check for expired sessions and report of the changes to session last activity time report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:3000}" json: # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON @@ -762,11 +857,14 @@ transport: # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" client_side_rpc: + # Processing timeout interval of the RPC command on the CLIENT SIDE. Time in milliseconds timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}" # Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property) api_enabled: "${TB_TRANSPORT_API_ENABLED:true}" log: + # Enable/Disable log of transport messages to telemetry. For example, logging of LwM2M registration update enabled: "${TB_TRANSPORT_LOG_ENABLED:true}" + # Maximum length of the log message. The content will be truncated to the specified value if needed max_length: "${TB_TRANSPORT_LOG_MAX_LENGTH:1024}" rate_limits: # Enable or disable generic rate limits. Device and Tenant specific rate limits are controlled in Tenant Profile. @@ -777,25 +875,36 @@ transport: ip_block_timeout: "${TB_TRANSPORT_IP_BLOCK_TIMEOUT:60000}" # Local HTTP transport parameters http: + # Enable/Disable local HTTP transport protocol enabled: "${HTTP_ENABLED:true}" + # HTTP request processing timeout in milliseconds request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}" + # HTTP maximum request processing timeout in milliseconds max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}" # Local MQTT transport parameters mqtt: # Enable/disable mqtt transport protocol. enabled: "${MQTT_ENABLED:true}" + # MQTT bind address bind_address: "${MQTT_BIND_ADDRESS:0.0.0.0}" + # MQTT bind port bind_port: "${MQTT_BIND_PORT:1883}" # Enable proxy protocol support. Disabled by default. If enabled, supports both v1 and v2. # Useful to get the real IP address of the client in the logs and for rate limits. proxy_enabled: "${MQTT_PROXY_PROTOCOL_ENABLED:false}" + # MQTT processing timeout in milliseconds timeout: "${MQTT_TIMEOUT:10000}" msg_queue_size_per_device_limit: "${MQTT_MSG_QUEUE_SIZE_PER_DEVICE_LIMIT:100}" # messages await in the queue before device connected state. This limit works on low level before TenantProfileLimits mechanism netty: + # Netty leak detector level leak_detector_level: "${NETTY_LEAK_DETECTOR_LVL:DISABLED}" + # Netty BOSS threads count boss_group_thread_count: "${NETTY_BOSS_GROUP_THREADS:1}" + # Netty worker threads count worker_group_thread_count: "${NETTY_WORKER_GROUP_THREADS:12}" + # Max payload size in bytes max_payload_size: "${NETTY_MAX_PAYLOAD_SIZE:65536}" + # Enables TCP keepalive. This means that TCP starts sending keepalive probes when a connection is idle for some time so_keep_alive: "${NETTY_SO_KEEPALIVE:false}" # MQTT SSL configuration ssl: @@ -837,11 +946,17 @@ transport: coap: # Enable/disable coap transport protocol. enabled: "${COAP_ENABLED:true}" + # CoAP bind address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" + # CoAP bind port bind_port: "${COAP_BIND_PORT:5683}" + # CoaP processing timeout in milliseconds timeout: "${COAP_TIMEOUT:10000}" + # CoaP piggyback response timeout in milliseconds piggyback_timeout: "${COAP_PIGGYBACK_TIMEOUT:500}" + # Default PSM Activity Timer if not specified in device profile psm_activity_timer: "${COAP_PSM_ACTIVITY_TIMER:10000}" + # Default PSM Activity Timer if not specified in device profile paging_transmission_window: "${COAP_PAGING_TRANSMISSION_WINDOW:10000}" dtls: # Enable/disable DTLS 1.2 support @@ -879,7 +994,9 @@ transport: x509: # Skip certificate validity check for client certificates. skip_validity_check_for_client_cert: "${TB_COAP_X509_DTLS_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" + # Inactivity timeout of DTLS session. Used to cleanup cache dtls_session_inactivity_timeout: "${TB_COAP_X509_DTLS_SESSION_INACTIVITY_TIMEOUT:86400000}" + # Interval of periodic eviction of the timed-out DTLS sessions dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Local LwM2M transport parameters lwm2m: @@ -889,11 +1006,16 @@ transport: # RFC7925_RETRANSMISSION_TIMEOUT_IN_MILLISECONDS = 9000 retransmission_timeout: "${LWM2M_DTLS_RETRANSMISSION_TIMEOUT_MS:9000}" server: + # LwM2M Server ID id: "${LWM2M_SERVER_ID:123}" + # LwM2M server bind address. Bind to all interfaces by default bind_address: "${LWM2M_BIND_ADDRESS:0.0.0.0}" + # LwM2M server bind port bind_port: "${LWM2M_BIND_PORT:5685}" security: + # LwM2M server bind address for DTLS. Bind to all interfaces by default bind_address: "${LWM2M_SECURITY_BIND_ADDRESS:0.0.0.0}" + # LwM2M server bind port for DTLS bind_port: "${LWM2M_SECURITY_BIND_PORT:5686}" # Server X509 Certificates support credentials: @@ -924,12 +1046,18 @@ transport: # Only Certificate_x509: skip_validity_check_for_client_cert: "${TB_LWM2M_SERVER_SECURITY_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" bootstrap: + # Enable/disable Bootstrap Server enabled: "${LWM2M_ENABLED_BS:true}" + # Default value in Lwm2mClient after start in mode Bootstrap for the object : name "LWM2M Security" field: "Short Server ID" (deviceProfile: Bootstrap.BOOTSTRAP SERVER.Short ID) id: "${LWM2M_SERVER_ID_BS:111}" + # LwM2M bootstrap server bind address. Bind to all interfaces by default bind_address: "${LWM2M_BS_BIND_ADDRESS:0.0.0.0}" + # LwM2M bootstrap server bind port bind_port: "${LWM2M_BS_BIND_PORT:5687}" security: + # LwM2M bootstrap server bind address for DTLS. Bind to all interfaces by default bind_address: "${LWM2M_BS_SECURITY_BIND_ADDRESS:0.0.0.0}" + # LwM2M bootstrap server bind address for DTLS. Bind to all interfaces by default bind_port: "${LWM2M_BS_SECURITY_BIND_PORT:5688}" # Bootstrap server X509 Certificates support credentials: @@ -976,103 +1104,167 @@ transport: store_file: "${LWM2M_TRUST_KEY_STORE:lwm2mtruststorechain.jks}" # Password used to access the key store store_password: "${LWM2M_TRUST_KEY_STORE_PASSWORD:server_ks_password}" + # Set usage of recommended cipher suites; true - allow only recommended cipher suites; false - allow not recommended cipher suites recommended_ciphers: "${LWM2M_RECOMMENDED_CIPHERS:false}" + # Set usage of recommended supported groups (curves); true - allow only recommended supported groups, false - allow not recommended supported groups recommended_supported_groups: "${LWM2M_RECOMMENDED_SUPPORTED_GROUPS:true}" + # Timeout of LwM2M operation timeout: "${LWM2M_TIMEOUT:120000}" + # Thread pool size for processing of the LwM2M uplinks uplink_pool_size: "${LWM2M_UPLINK_POOL_SIZE:10}" + # Thread pool size for processing of the LwM2M downlinks downlink_pool_size: "${LWM2M_DOWNLINK_POOL_SIZE:10}" + # Thread pool size for processing of the OTA updates ota_pool_size: "${LWM2M_OTA_POOL_SIZE:10}" + # Period of cleanup for the registrations in store clean_period_in_sec: "${LWM2M_CLEAN_PERIOD_IN_SEC:2}" + # Maximum log size log_max_length: "${LWM2M_LOG_MAX_LENGTH:1024}" + # PSM Activity Timer if not specified in device profile psm_activity_timer: "${LWM2M_PSM_ACTIVITY_TIMER:10000}" + # Paging Transmission Window for eDRX support if not specified in the device profile paging_transmission_window: "${LWM2M_PAGING_TRANSMISSION_WINDOW:10000}" network_config: # In this section you can specify custom parameters for LwM2M network configuration and expose the env variables to configure outside # - key: "PROTOCOL_STAGE_THREAD_COUNT" # value: "${LWM2M_PROTOCOL_STAGE_THREAD_COUNT:4}" snmp: + # Enable/disable SNMP transport protocol enabled: "${SNMP_ENABLED:true}" + # Snmp bind port bind_port: "${SNMP_BIND_PORT:1620}" response_processing: # parallelism level for executor (workStealingPool) that is responsible for handling responses from SNMP devices parallelism_level: "${SNMP_RESPONSE_PROCESSING_PARALLELISM_LEVEL:20}" # to configure SNMP to work over UDP or TCP underlying_protocol: "${SNMP_UNDERLYING_PROTOCOL:udp}" + # Batch size to request OID mappings from device (useful when device profile has multiple hundreds of OID mappings) max_request_oids: "${SNMP_MAX_REQUEST_OIDS:100}" response: + # To ignore SNMP response values that do not match data type of the configured OID mapping (by default false - will throw error if any value of the response not matches configured data types) ignore_type_cast_errors: "${SNMP_RESPONSE_IGNORE_TYPE_CAST_ERRORS:false}" stats: + # Enable/Disable collection of transport statistics enabled: "${TB_TRANSPORT_STATS_ENABLED:true}" + # Interval of transport statistics logging print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Device connectivity properties to publish telemetry device: connectivity: http: + # If true check-connectivity service will include curl command to the list of all test commands using DEVICE_CONNECTIVITY_HTTP_HOST and DEVICE_CONNECTIVITY_HTTP_PORT variables enabled: "${DEVICE_CONNECTIVITY_HTTP_ENABLED:true}" + # Host of http transport service. If empty base url will be used. host: "${DEVICE_CONNECTIVITY_HTTP_HOST:}" + # Port of http transport service. If empty default http port will be used. port: "${DEVICE_CONNECTIVITY_HTTP_PORT:8080}" https: + # If true check-connectivity service will include curl command to the list of all test commands using DEVICE_CONNECTIVITY_HTTPS_HOST and DEVICE_CONNECTIVITY_HTTPS_PORT variables enabled: "${DEVICE_CONNECTIVITY_HTTPS_ENABLED:false}" + # Host of http transport service. If empty base url will be used. host: "${DEVICE_CONNECTIVITY_HTTPS_HOST:}" + # Port of http transport service. If empty default https port will be used. port: "${DEVICE_CONNECTIVITY_HTTPS_PORT:443}" mqtt: + # If true mosquito command will be included to the list of all test commands using DEVICE_CONNECTIVITY_MQTT_HOST and DEVICE_CONNECTIVITY_MQTT_PORT enabled: "${DEVICE_CONNECTIVITY_MQTT_ENABLED:true}" + # Host of mqtt transport service. If empty base url host will be used. host: "${DEVICE_CONNECTIVITY_MQTT_HOST:}" + # Port of mqtt transport service port: "${DEVICE_CONNECTIVITY_MQTT_PORT:1883}" mqtts: + # If true mosquito command will be included to the list of all test commands using DEVICE_CONNECTIVITY_MQTTS_HOST and DEVICE_CONNECTIVITY_MQTTS_PORT< enabled: "${DEVICE_CONNECTIVITY_MQTTS_ENABLED:false}" + # Host of mqtt transport service. If empty base url host will be used. host: "${DEVICE_CONNECTIVITY_MQTTS_HOST:}" + # Port of mqtt transport service. If empty default port for mqtts will be used. port: "${DEVICE_CONNECTIVITY_MQTTS_PORT:8883}" + # Path to the server certificate file pem_cert_file: "${DEVICE_CONNECTIVITY_MQTT_SSL_PEM_CERT:mqttserver.pem}" coap: + # If true coap command will be included to the list of all test commands using DEVICE_CONNECTIVITY_COAP_HOST and DEVICE_CONNECTIVITY_COAP_PORT. enabled: "${DEVICE_CONNECTIVITY_COAP_ENABLED:true}" + # Host of coap transport service. If empty base url host will be used. host: "${DEVICE_CONNECTIVITY_COAP_HOST:}" + # Port of coap transport service. If empty default port for coap will be used. port: "${DEVICE_CONNECTIVITY_COAP_PORT:5683}" coaps: + # If true coap command will be included to the list of all test commands using DEVICE_CONNECTIVITY_COAPS_HOST and DEVICE_CONNECTIVITY_COAPS_PORT. enabled: "${DEVICE_CONNECTIVITY_COAPS_ENABLED:false}" + # Host of coap transport service. If empty base url host will be used. host: "${DEVICE_CONNECTIVITY_COAPS_HOST:}" + # Port of coap transport service. If empty default port for coaps will be used. port: "${DEVICE_CONNECTIVITY_COAPS_PORT:5684}" # Edges parameters edges: + # Enable/disable Edge instance enabled: "${EDGES_ENABLED:true}" rpc: + # RPC port bind port: "${EDGES_RPC_PORT:7070}" + # Maximum time of keep alive client RPC connection. Time in seconds client_max_keep_alive_time_sec: "${EDGES_RPC_CLIENT_MAX_KEEP_ALIVE_TIME_SEC:1}" + # Maximum time of keep alive connection. Time in seconds keep_alive_time_sec: "${EDGES_RPC_KEEP_ALIVE_TIME_SEC:10}" + # Timeout of keep alive RPC connection. Time in seconds keep_alive_timeout_sec: "${EDGES_RPC_KEEP_ALIVE_TIMEOUT_SEC:5}" ssl: # Enable/disable SSL support enabled: "${EDGES_RPC_SSL_ENABLED:false}" + # Cert file to be used during TLS connectivity to cloud cert: "${EDGES_RPC_SSL_CERT:certChainFile.pem}" + # Private key file associated with the Cert certificate. This key is used in the encryption process during a secure connection private_key: "${EDGES_RPC_SSL_PRIVATE_KEY:privateKeyFile.pem}" + # Maximum size (in bytes) of inbound messages that cloud can handle from edge. By default, it can handle messages up to 4 Megabytes max_inbound_message_size: "${EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE:4194304}" storage: + # Max records of edge event to read from DB and sent to edge max_read_records_count: "${EDGES_STORAGE_MAX_READ_RECORDS_COUNT:50}" + # Number of milliseconds to wait before next check of edge events in DB no_read_records_sleep: "${EDGES_NO_READ_RECORDS_SLEEP:1000}" + # Number of milliseconds to wait before resend failed batch of edge events to edge sleep_between_batches: "${EDGES_SLEEP_BETWEEN_BATCHES:10000}" + # Number of threads that used to check DB for edge events scheduler_pool_size: "${EDGES_SCHEDULER_POOL_SIZE:1}" + # Number of threads that used to send downlink messages to edge over gRPC send_scheduler_pool_size: "${EDGES_SEND_SCHEDULER_POOL_SIZE:1}" + # Number of threads that used to convert edge events from DB into downlink message and send them for delivery grpc_callback_thread_pool_size: "${EDGES_GRPC_CALLBACK_POOL_SIZE:1}" state: + # Persist state of edge (active, last connect, last disconnect) into timeseries or attributes tables. 'false' means to store edge state into attributes table persistToTelemetry: "${EDGES_PERSIST_STATE_TO_TELEMETRY:false}" +# Swagger common properties swagger: + # If false swagger api docs will be unavailable enabled: "${SWAGGER_ENABLED:true}" + # General swagger match pattern of swaggerUI links api_path_regex: "${SWAGGER_API_PATH_REGEX:/api/.*}" + # General swagger match pattern path of swaggerUI links security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api/.*}" + # Non security API path match pattern of swaggerUI links non_security_path_regex: "${SWAGGER_NON_SECURITY_PATH_REGEX:/api/(?:noauth|v1)/.*}" + # The title on the API doc UI page title: "${SWAGGER_TITLE:ThingsBoard REST API}" + # The description on the API doc UI page description: "${SWAGGER_DESCRIPTION: ThingsBoard open-source IoT platform REST API documentation.}" contact: + # The contact name on the API doc UI page name: "${SWAGGER_CONTACT_NAME:ThingsBoard team}" + # The contact URL on the API doc UI page url: "${SWAGGER_CONTACT_URL:https://thingsboard.io}" + # The contact email on the API doc UI page email: "${SWAGGER_CONTACT_EMAIL:info@thingsboard.io}" license: + # The license title on the API doc UI page title: "${SWAGGER_LICENSE_TITLE:Apache License Version 2.0}" + # Link to the license body on the API doc UI page url: "${SWAGGER_LICENSE_URL:https://github.com/thingsboard/thingsboard/blob/master/LICENSE}" + # The version of the API doc to display. Default to the package version. version: "${SWAGGER_VERSION:}" +# Queue configuration parameters queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) in_memory: @@ -1080,44 +1272,73 @@ queue: # For debug lvl print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" kafka: + # Kafka Bootstrap Servers bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" ssl: + # Enable/Disable SSL Kafka communication enabled: "${TB_KAFKA_SSL_ENABLED:false}" + # The location of the trust store file truststore.location: "${TB_KAFKA_SSL_TRUSTSTORE_LOCATION:}" + # The password of trust store file if specified truststore.password: "${TB_KAFKA_SSL_TRUSTSTORE_PASSWORD:}" + # The location of the key store file. This is optional for client and can be used for two-way authentication for client keystore.location: "${TB_KAFKA_SSL_KEYSTORE_LOCATION:}" + # The store password for the key store file. This is optional for client and only needed if ‘ssl.keystore.location’ is configured. Key store password is not supported for PEM format keystore.password: "${TB_KAFKA_SSL_KEYSTORE_PASSWORD:}" + # The password of the private key in the key store file or the PEM key specified in ‘keystore.key’ key.password: "${TB_KAFKA_SSL_KEY_PASSWORD:}" + # The number of acknowledgments the producer requires the leader to have received before considering a request complete. This controls the durability of records that are sent. The following settings are allowed:0,1 and all acks: "${TB_KAFKA_ACKS:all}" + # Number of retries. Resend any record whose send fails with a potentially transient error retries: "${TB_KAFKA_RETRIES:1}" + # The compression type for all data generated by the producer. The default is none (i.e. no compression). Valid values none or gzip compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + # Default batch size. This setting gives the upper bound of the batch size to be sent batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + # This variable creates a small amount of artificial delay—that is, rather than immediately sending out a record linger.ms: "${TB_KAFKA_LINGER_MS:1}" + # The maximum size of a request in bytes. This setting will limit the number of record batches the producer will send in a single request to avoid sending huge requests max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + # The maximum number of unacknowledged requests the client will send on a single connection before blocking max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + # The total bytes of memory the producer can use to buffer records waiting to be sent to the server buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + # The multiple copies of data over the multiple brokers of Kafka replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + # The maximum delay between invocations of poll() when using consumer group management. This places an upper bound on the amount of time that the consumer can be idle before fetching more records max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + # The maximum number of records returned in a single call to poll() max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + # The maximum amount of data per-partition the server will return. Records are fetched in batches by the consumer max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + # The maximum amount of data the server will return. Records are fetched in batches by the consumer fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none + # Enable/Disable using of Confluent Cloud use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: + # The endpoint identification algorithm used by clients to validate server host name. The default value is https ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + # The mechanism used to authenticate Schema Registry requests. SASL/PLAIN should only be used with TLS/SSL as transport layer to ensure that clear passwords are not transmitted on the wire without encryption sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + # Using JAAS Configuration for specifying multiple SASL mechanisms on a broker sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + # Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params consumer-properties-per-topic: tb_ota_package: + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params - key: max.poll.records + # Example of specific consumer properties value per topic value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" tb_version_control: + # Example of specific consumer properties value per topic for VC - key: max.poll.interval.ms + # Example of specific consumer properties value per topic for VC value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records @@ -1129,102 +1350,182 @@ queue: # - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms # value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: + # Kafka properties for Rule Engine rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Core topics core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Transport Api topics transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for JS Executor topics js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + # Kafka properties for OTA updates topic ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: + # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + # Statistics printing interval for Kafka's consumer-groups stats print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + # Time to wait for the stats-loading requests to Kafka to finis kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" aws_sqs: + # Use default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + # Access key ID from AWS IAM user access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + # Secret access key from AWS IAM user secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + # Region from AWS account region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + # Number of threads per each AWS SQS queue in consumer threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" queue-properties: + # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800 js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds ota-updates: "${TB_QUEUE_AWS_SQS_OTA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: + # Project ID from Google Cloud project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + # API Credentials in JSON format service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" - max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + # Message size for PubSub queue.Value in bytes + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" + # Number of messages per a consumer max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" queue-properties: + # Pub/Sub properties for Rule Engine subscribers, messages which will commit after ackDeadlineInSec period can be consume again rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + # Pub/Sub properties for Core subscribers, messages which will commit after ackDeadlineInSec period can be consume again core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + # Pub/Sub properties for Transport API subscribers, messages which will commit after ackDeadlineInSec period can be consume again transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + # Pub/Sub properties for Version Control subscribers, messages which will commit after ackDeadlineInSec period can be consume again notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + # PubSub queue properties js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + # Pub/Sub properties for Transport Api subscribers, messages which will commit after ackDeadlineInSec period can be consume again version-control: "${TB_QUEUE_PUBSUB_VC_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: + # Azure namespace namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + # Azure Service Bus Shared Access Signatures key name sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + # Azure Service Bus Shared Access Signatures key sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + # Number of messages per a consumer max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" queue-properties: + # Azure Service Bus properties for Rule Engine queues rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + # Azure Service Bus properties for Core queues core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + # Azure Service Bus properties for Transport Api queues transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + # Azure Service Bus properties for Notification queues notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + # Azure Service Bus queue properties js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + # Azure Service Bus properties for Version Control queues version-control: "${TB_QUEUE_SERVICE_BUS_VC_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: + # By default empty exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + # RabbitMQ host used to establish connection host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + # RabbitMQ host used to establish connection port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + # Virtual hosts provide logical grouping and separation of resources virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + # Username for RabbitMQ user account username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + # User password for RabbitMQ user account password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + # Network connection between clients and RabbitMQ nodes can fail. RabbitMQ Java client supports automatic recovery of connections and topology (queues, exchanges, bindings, and consumers) automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + # The connection timeout for the RabbitMQ connection factory connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + # RabbitMQ has a timeout for connection handshake. When clients run in heavily constrained environments, it may be necessary to increase the timeout handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" queue-properties: + # RabbitMQ properties for Rule Engine queues rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + # RabbitMQ properties for Core queues core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + # RabbitMQ properties for Transport API queues transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + # RabbitMQ properties for Notification queues notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + # RabbitMQ queue properties js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + # RabbitMQ properties for Version Control queues version-control: "${TB_QUEUE_RABBIT_MQ_VC_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: + # Topic used to consume api requests from transport microservices requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + # Topic used to produce api responses to transport microservices responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + # Maximum pending api requests from transport microservices to be handled by server max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + # Maximum timeout in milliseconds to handle api request from transport microservice by server max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + # Amount of threads used to invoke callbacks max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + # Interval in milliseconds to poll api requests from transport microservices request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + # Interval in milliseconds to poll api response from transport microservices response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" core: + # Default topic name of Kafka, RabbitMQ, etc. queue topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + # Interval in milliseconds to poll messages by Core microservices poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + # Amount of partitions used by Core microservices partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + # Timeout for processing a message pack by Core microservices pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" ota: + # Default topic name for OTA updates topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + # The interval of processing the OTA updates for devices. Used to avoid any harm to network due to many parallel OTA updates pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + # The size of OTA updates notifications fetched from the queue. The queue stores pairs of firmware and device ids pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + # Default topic name for queue Kafka, RabbitMQ, etc. usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: + # Enable/disable statistics for Core microservices enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" vc: + # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + # Number of partitions to associate with this queue. Used for scaling the number of messages that can be processed in parallel partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + # Interval in milliseconds between polling of the messages if no new messages arrive poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + # Timeout before retry all failed and timed-out messages from processing pack pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:180000}" + # Timeout for a request to VC-executor (for a request for the version of the entity, for a commit charge, etc.) request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:180000}" + # Queue settings for Kafka, RabbitMQ, etc. Limit for single message size msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" js: # JS Eval request topic @@ -1242,19 +1543,24 @@ queue: # JS response poll interval response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" rule-engine: + # Deprecated. Will be removed in nearest releases topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + # Interval in milliseconds to poll messages by Rule Engine poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + # Timeout for processing a message pack of Rule Engine pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" stats: + # Enable/disable statistics for Rule Engine enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + # Statistics printing interval for Rule Engine print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" queues: - - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" - topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" - poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - consumer-per-partition: "${TB_QUEUE_RE_MAIN_CONSUMER_PER_PARTITION:true}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}" + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" # queue name + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" # queue topic + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" # poll interval + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" # number queue partitions + consumer-per-partition: "${TB_QUEUE_RE_MAIN_CONSUMER_PER_PARTITION:true}" # if true - use for each customer different partition + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}" # Timeout for processing a message pack submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -1266,12 +1572,12 @@ queue: failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}" # Max allowed time in seconds for pause between retries. - - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" - topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" - poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" - consumer-per-partition: "${TB_QUEUE_RE_HP_CONSUMER_PER_PARTITION:true}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}" + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" # queue name + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" # queue topic + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" # poll interval + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" # number queue partitions + consumer-per-partition: "${TB_QUEUE_RE_HP_CONSUMER_PER_PARTITION:true}" # if true - use for each customer different partition + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}" # Timeout for processing a message pack submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -1283,12 +1589,12 @@ queue: failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. - - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" - topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" - poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" - partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" - consumer-per-partition: "${TB_QUEUE_RE_SQ_CONSUMER_PER_PARTITION:true}" - pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}" + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" # queue name + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" # queue topic + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" # poll interval + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" # number queue partitions + consumer-per-partition: "${TB_QUEUE_RE_SQ_CONSUMER_PER_PARTITION:true}" # if true - use for each customer different partition + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}" # Timeout for processing a message pack submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -1305,12 +1611,16 @@ queue: transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + # Interval in milliseconds to poll messages poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" +# Event configuration parameters event: debug: + # Maximum number of symbols per debug event. The event content will be truncated if needed max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" +# General service parameters service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) @@ -1320,6 +1630,7 @@ service: # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in profile). assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" +# Metrics parameters metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" @@ -1327,24 +1638,28 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +# Version control parameters vc: # Pool size for handling export tasks thread_pool_size: "${TB_VC_POOL_SIZE:2}" git: # Pool size for handling the git IO operations io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + # Default storing repository path repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" +# Notification system parameters notification_system: + # Specify thread pool size for Notification System processing notification rules and notification sending. Recommend value < =10 thread_pool_size: "${TB_NOTIFICATION_SYSTEM_THREAD_POOL_SIZE:10}" rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' deduplication_durations: "${TB_NOTIFICATION_RULES_DEDUPLICATION_DURATIONS:NEW_PLATFORM_VERSION:0;RATE_LIMITS:14400000;}" +# General management parameters management: endpoints: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' - + include: '${METRICS_ENDPOINTS_EXPOSE:info}' \ No newline at end of file diff --git a/pull_request_template.md b/pull_request_template.md index f65aa0a1f3..584ee6f4fc 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -26,6 +26,7 @@ Put your PR description here instead of this sentence. - [ ] If new dependency was added: the dependency tree is checked for conflicts. - [ ] If new service was added: the service is marked with corresponding @TbCoreComponent, @TbRuleEngineComponent, @TbTransportComponent, etc. - [ ] If new REST API was added: the RestClient.java was updated, issue for [Python REST client](https://github.com/thingsboard/thingsboard-python-rest-client) is created. +- [ ] If new yml property was added: make sure a description is added (above or near the property). diff --git a/tools/src/main/python/check_yml_file.py b/tools/src/main/python/check_yml_file.py new file mode 100644 index 0000000000..2c65876f70 --- /dev/null +++ b/tools/src/main/python/check_yml_file.py @@ -0,0 +1,105 @@ +import sys +import re + +def extract_properties_with_comments(yaml_file_path): + properties = {} + + with open(yaml_file_path, 'r') as file: + lines = file.readlines() + index = 0 + key_level_map = {0 : ''} + parse_line('', '', key_level_map, 0, index, lines, properties) + + return properties + + +def parse_line(table_name, comment, key_level_map, parent_line_level, index, lines, properties): + if index >= len(lines): + return + line = lines[index] + line_level = (len(line) - len(line.lstrip())) if line.strip() else 0 + if line_level == 0: + key_level_map = {0 : ''} + line = line.strip() + # if line is empty - parse next line + if not line: + index = index + 1 + parse_line(table_name, comment, key_level_map, line_level, index, lines, properties) + # if line is a comment - save comment and parse next line + elif line.startswith('#'): + if line_level == 0: + table_name = line.lstrip('#') + elif line_level == parent_line_level: + comment = comment + '\n' + line.lstrip('#') + else: + comment = line.lstrip('#') + index = index + 1 + parse_line(table_name, comment, key_level_map, line_level, index, lines, properties) + else: + # Check if it's a property line + if ':' in line: + # clean comment if level was changed + if line_level != parent_line_level: + comment = '' + key, value = line.split(':', 1) + if key.startswith('- '): + key = key.lstrip('- ') + key_level_map[line_level] = key + value = value.strip() + if value.split('#')[0]: + current_key = '' + for k in key_level_map.keys(): + if k <= line_level: + current_key = ((current_key + '.') if current_key else '') + key_level_map[k] + properties[current_key] = (value, comment, table_name) + comment = '' + index = index + 1 + parse_line(table_name, comment, key_level_map, line_level, index, lines, properties) + +def extract_property_info(properties): + rows = [] + for property_name, value in properties.items(): + if '#' in value[0]: + value_parts = value[0].split('#') + comment = value_parts[1] + else: + comment = value[1] + pattern = r'\"\$\{(.*?)\:(.*?)\}\"' + match = re.match(pattern, value[0]) + if match is not None: + rows.append((property_name, match.group(1), match.group(2), comment, value[2])) + else: + rows.append((property_name, "", value[0].split('#')[0], comment, value[2])) + return rows + +def check_descriptions(properties): + variables_without_description = [] + for row in properties: + # Extract information from the tuple + property_name, env_variable, default_value, comment, table_name = row + if comment == '' or len(comment) < 5 : + variables_without_description.append(property_name) + + return variables_without_description + + +if __name__ == '__main__': + sys. setrecursionlimit(10000) + # path to the YAML file + input_yaml_file = "application/src/main/resources/thingsboard.yml" + + # Parse yml file to map where key is property key path with '.' separator + # and value is an object (env_name_with_default_value, comment, table_name) + properties = extract_properties_with_comments(input_yaml_file) + + # Extract property information (extract env name, default value and comment nearby property) + property_info = extract_property_info(properties) + + # Check all properies have descriptions + variables_without_desc = check_descriptions(property_info) + + if len(variables_without_desc) > 0: + print(f"There are some yml properties without valid description: (total {len(variables_without_desc)}) {variables_without_desc}.") + exit(1) + else: + print("All yml properties have valid description.") \ No newline at end of file From a7daf55a38d291f35e857ec6728865e97c94dcc4 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Oct 2023 14:43:42 +0300 Subject: [PATCH 19/36] fixed github action yml file --- ...{check_yml_parameter_descriptions.yml => check_yml_file.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{check_yml_parameter_descriptions.yml => check_yml_file.yml} (88%) diff --git a/.github/workflows/check_yml_parameter_descriptions.yml b/.github/workflows/check_yml_file.yml similarity index 88% rename from .github/workflows/check_yml_parameter_descriptions.yml rename to .github/workflows/check_yml_file.yml index 240222d1b1..3ec0a64439 100644 --- a/.github/workflows/check_yml_parameter_descriptions.yml +++ b/.github/workflows/check_yml_file.yml @@ -21,4 +21,4 @@ jobs: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Run Verification Script - run: python3 tools/src/main/python/check_env_variables.py + run: python3 tools/src/main/python/check_yml_file.py From 12f140c5e28046e4cdc03cde7ab2fa203276ef83 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Oct 2023 14:49:48 +0300 Subject: [PATCH 20/36] renamed github action yml file --- .github/workflows/{check_yml_file.yml => main.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/{check_yml_file.yml => main.yml} (89%) diff --git a/.github/workflows/check_yml_file.yml b/.github/workflows/main.yml similarity index 89% rename from .github/workflows/check_yml_file.yml rename to .github/workflows/main.yml index 3ec0a64439..f420254f34 100644 --- a/.github/workflows/check_yml_file.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,11 @@ -name: Check yml parameters have description +name: check_yml on: pull_request: paths: - 'application/src/main/resources/thingsboard.yml' jobs: build: - name: check + name: Check yml parameters have description runs-on: ubuntu-20.04 steps: From 5d494f81dd171a4022a78058271afc88cc4c8fab Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Oct 2023 15:25:05 +0300 Subject: [PATCH 21/36] renamed github action job name --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f420254f34..5a4d79428c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,11 @@ -name: check_yml +name: check_yml_file on: pull_request: paths: - 'application/src/main/resources/thingsboard.yml' jobs: build: - name: Check yml parameters have description + name: check_yml_file runs-on: ubuntu-20.04 steps: From 4c00b839e687d2119a84cc7e6f5167e8c2de9bbc Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 5 Oct 2023 17:33:06 +0300 Subject: [PATCH 22/36] add yml parameter description --- application/src/main/resources/thingsboard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a94ecc8de0..b73c4f1fda 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1554,6 +1554,7 @@ queue: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" # Statistics printing interval for Rule Engine print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + # Max length of error message that is printed by statistics max-error-message-length: "${TB_QUEUE_RULE_ENGINE_MAX_ERROR_MESSAGE_LENGTH:4096}" queues: - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" # queue name From 2492b81d3403d5de852c2eba7e0c93522db9ce78 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 5 Oct 2023 17:52:49 +0300 Subject: [PATCH 23/36] UI: Wind speed and direction widget. --- .../widget_bundles/weather_widgets.json | 13 + .../wind_speed_and_direction.json | 29 ++ .../DefaultSystemDataLoaderService.java | 1 + .../sql/widget/WidgetTypeInfoRepository.java | 2 +- ui-ngx/package.json | 1 + ui-ngx/src/app/core/utils.ts | 4 +- .../basic/basic-widget-config.module.ts | 12 +- .../basic/common/data-key-row.component.html | 37 +- .../basic/common/data-key-row.component.scss | 25 ++ .../basic/common/data-key-row.component.ts | 138 +++++--- .../common/data-keys-panel.component.html | 9 + .../basic/common/data-keys-panel.component.ts | 17 - ...peed-direction-basic-config.component.html | 213 +++++++++++ ...-speed-direction-basic-config.component.ts | 291 +++++++++++++++ .../battery-level-widget.component.html | 4 +- .../battery-level-widget.component.scss | 214 ++++++----- .../battery-level-widget.component.ts | 6 +- ...d-direction-widget-settings.component.html | 102 ++++++ ...eed-direction-widget-settings.component.ts | 139 ++++++++ .../lib/settings/widget-settings.module.ts | 12 +- ...wind-speed-direction-widget.component.html | 25 ++ ...wind-speed-direction-widget.component.scss | 49 +++ .../wind-speed-direction-widget.component.ts | 333 ++++++++++++++++++ .../wind-speed-direction-widget.models.ts | 108 ++++++ .../widget/widget-components.module.ts | 9 +- .../shared/models/widget-settings.models.ts | 41 +++ .../assets/locale/locale.constant-en_US.json | 21 ++ .../wind-speed-direction/advanced-layout.svg | 187 ++++++++++ .../wind-speed-direction/default-layout.svg | 167 +++++++++ .../simplified-layout.svg | 153 ++++++++ ui-ngx/src/form.scss | 12 +- ui-ngx/yarn.lock | 5 + 32 files changed, 2167 insertions(+), 212 deletions(-) create mode 100644 application/src/main/data/json/system/widget_bundles/weather_widgets.json create mode 100644 application/src/main/data/json/system/widget_types/wind_speed_and_direction.json create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/weather/wind-speed-direction-basic-config.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/config/basic/weather/wind-speed-direction-basic-config.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/weather/wind-speed-direction-widget.component.html create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/weather/wind-speed-direction-widget.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/weather/wind-speed-direction-widget.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/widget/lib/weather/wind-speed-direction-widget.models.ts create mode 100644 ui-ngx/src/assets/widget/wind-speed-direction/advanced-layout.svg create mode 100644 ui-ngx/src/assets/widget/wind-speed-direction/default-layout.svg create mode 100644 ui-ngx/src/assets/widget/wind-speed-direction/simplified-layout.svg diff --git a/application/src/main/data/json/system/widget_bundles/weather_widgets.json b/application/src/main/data/json/system/widget_bundles/weather_widgets.json new file mode 100644 index 0000000000..a38c139cf0 --- /dev/null +++ b/application/src/main/data/json/system/widget_bundles/weather_widgets.json @@ -0,0 +1,13 @@ +{ + "widgetsBundle": { + "alias": "weather_widgets", + "title": "Weather widgets", + "image": null, + "description": null, + "externalId": null, + "name": "Weather widgets" + }, + "widgetTypeFqns": [ + "wind_speed_and_direction" + ] +} \ No newline at end of file diff --git a/application/src/main/data/json/system/widget_types/wind_speed_and_direction.json b/application/src/main/data/json/system/widget_types/wind_speed_and_direction.json new file mode 100644 index 0000000000..1c4f7ecb7c --- /dev/null +++ b/application/src/main/data/json/system/widget_types/wind_speed_and_direction.json @@ -0,0 +1,29 @@ +{ + "fqn": "wind_speed_and_direction", + "name": "Wind speed and direction", + "deprecated": false, + "image": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAACgCAMAAAB+IdObAAAAilBMVEXg4ODf39/g4OAAAADg4ODf39/////g4OD7+/v39/fn5+f09PTx8fEhISHr6+vu7u7k5OSenp7i4uLOzs6qqqqQkJDt7e3IyMjw8PDp6em2trbV1dU8PDwvLy/b29vCwsJXV1eCgoKGhoa6urptbW10dHRKSkqkpKRmZma8vLywsLCSkpJ7e3thYWHl47mIAAAABnRSTlPvIL8Ar7DvmsykAAAHPklEQVR42uzZbY+aQBSGYfuSh+ecmWFgRFBBq2m2u9v0//++grtGq9uUioIa70Txk3o5c8wYR18+fR5Ft97XT19Gn0aeuPFoa8ZXizvI19vq5tejiaNRhLsoekCurAfk2npArq0H5Np6QK6tB+TaekDaV4wBfcVhtweZp4plhsNuEJJlch+Q8Xx8JxDzPL4PCMbpnUAkuweI5oA5fpXbg3zYA/KAdM2ABoawILHfDUEsrIVSYmQlFd4ggRjsGhRCbm7k9tGHcfO2G0NMmnH6bD2sUN9oJDA4JFHAK9TCBEINjhNBgtpgCKLOpGk6f/eZRmM8ZHhIMA0kjxEHYcRjBoyHFxLbsrRuiW1ELkyOF5MqgJ3CaB0vD/GBXiEJknha3x9EqDDGfnnaNP9T6xEbEPsxSog4hlcRweUhMvNewUgSUTs7dOgH4yymTnDQDBIfQvwGEgPoA8KgCugsRlCL/cwUOdE6TyTch0iQBpKoTvuAwEYK5JGBBsEuSyZo2U5uuYPQ6vvWYi8QqAImImzYHw47w39HKrmDQEPHrdU9FRicIBHEZgeRqNlacRxzIIgFc5wWPbih0BAwAjF1Q0Fyj9OjEryKs9aMkC4QQbS8AogVmaFbSWE5PKQsBd2KoynVDAwhWEbdJD6awkzBYSF2RmroIAlPv9ZZUQRRDgkhLEFVnJxdVa5pTchwEFHZcIhO2bVzE/jZYBCCBmdo4lYuhQE5EESnOEP2ya2D+w4wMQNBYNG98MOlKFwBiGAQiCRE98pqUaCGlKizOgQEtOhctqhCc3EWAD0HgFhzljF/sagrXrCJsfQOYSxnGPMJ/sgb9A0RnmfMDxPpGWKUnce8Kj942p4hPNeYH8deIUl8ljE/ThLb84ycdcx3kegR0vl3UOZS/CVOfX8QSI5uBfwtsewPIrhkZG+QWNnmUy/CKWdMr+aq/noLK+fcKuwU6WqClvUFmbY5nVTuKXtxK7xVNqyWEKPSE4QtzhGFqwC7cAGbJounl9YQQQ+QtsP43f0AUG0h1mLSFgKyJ8gsxj+zi0WoNRW2tYdIkl/TsJcLV7kfYR9ybcPuvbQ6TC1WlZucAjGx9AKhsM2MVBZh4YoTICLsB4IWrV0K4MmlJ0DAXiDUhG0gawAvJ0FsYi8FGRvkJVAatK1wbvJ97ZqvrkV6yrCbcV1+bsi8RPEMSXMA9K04aeWcWxRA6lb/CaG3gKaXgBTf8DNTfUYdjUGriqx8u1o02WDbQoxpIJfYWlFmnouynKNOtMngQuXa9L4iZ4dIunzV+WuJumXatMSFmqd12aUgyH63d647TkJRGPW62Bc8UEAKVVutjcYZ5/1fz3aMoU47sQjDZdKV9ld/NCvsDw6cfQ43ay0KA3A7oDT0f0TKpyotNgtntWgq+EkR14PIer3uPexEG4jy5pzypEiSgG/2RBMYNHZjOlf2LsQWT2esdZZpjbUQ/1+ViY1+y6C0ZGvtuhB0QneIYL429h+wHXhON/oX0VBzCbuft5+3t59XsFqxuVvcORehVjGdO0Rgt+a2YLOFHWwLcrv4D7qI9G+yMxYr8i3rLdh2t3UuQoWBRMqgrUS2a1g529WllxEd/3HQYnlG5CdQ7BZ3RsMUwg5Jqpzly/FdoIMfvvkGwHLnIhLTwURcHum9evd+XvMjaMUp2cdPX+mMuzCYiNTOCcvDfHN3JCTjzuouDg962zCFWV3E7OwDue5oWg0pgvQS8wl0PuDWY8wb4iAMLCJ9xrzBfWARpNKTmHemchhaJMS9xny8DjpQJ+s15sTA8CJu+vXT10OTaG8kNoYIYN/e9RdzVGAckTqJPn2mJ8bsxBbkpregj9kbj5j20cjsAlAGGE1EiEXSTOiCRibEgo66osf2Hko3PDJGXtGDSJorXfEPEcKoIlR5TXfiZTn68j1Fu6qooeOvQ4TQLacIiTO8iLiCKKggyj1J1XGt7j3qAqJ7hhDxNJhJbEikf079Uv7/tUQq1DlgVmcxITOzIUQsgdQ1w6OEpm0gdZT2iGAx92gkuBHCQKUVpy4gmYRgRNrUuQfaI812L5IGBUI9UGnhFlVgnkrm2bGhiAmt8AqXRiqkqd6XVjyEiAqSKWWdUlvNMXFF1UIl0b/MRaEcsLQsQTLHo0AcxScDcdQva+RNUOMYzYTECLWIDCEiFmUJSOr7r578Sq1UPKDIT1YkUCv6QC2NTCnTPVPojXc8kCjymIiQqKSowvjzI/9KkTW7OR2LCC5ixBUOTF4EUEr/bVOJ+F5EfGr7a7UgaXY8y8VIHJvQjmftEBxxlhElohwzL5FHuYpcRXrgw3LtnDA/kc3N+kfBCfMTKSJ0ozxkfiIfio1xyvxEKDfFkhNmJ6K54gvnIbMTocht+f05ZMSXq3MhmZ/IWa4iV5G5cBWZGleRqXEVmRpXkalxFZkaz0jkxfN4QXD04lXFMyB+u3/7dDz7YyLVXuP1yzfR3Hnx6uXrXzaOLHMeGQCLAAAAAElFTkSuQmCC", + "description": "Displays the latest values of the wind speed and direction.", + "descriptor": { + "type": "latest", + "sizeX": 3, + "sizeY": 3, + "resources": [], + "templateHtml": "\n", + "templateCss": "", + "controllerScript": "self.onInit = function() {\n self.ctx.$scope.windSpeedDirectionWidget.onInit();\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.windSpeedDirectionWidget.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\n maxDataKeys: 2,\n singleEntity: true,\n previewWidth: '270px',\n previewHeight: '270px',\n embedTitlePanel: true\n };\n};\n\nself.actionSources = function() {\n return {\n 'cardClick': {\n name: 'widget-action.card-click',\n multiple: false\n }\n };\n}\n\nself.onDestroy = function() {\n};\n", + "settingsSchema": "", + "dataKeySettingsSchema": "", + "settingsDirective": "tb-wind-speed-direction-widget-settings", + "hasBasicMode": true, + "basicModeDirective": "tb-wind-speed-direction-basic-config", + "defaultConfig": "{\"datasources\":[{\"type\":\"function\",\"name\":\"function\",\"dataKeys\":[{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"windDirection\",\"color\":\"#4caf50\",\"settings\":{},\"_hash\":0.7227918773301678,\"funcBody\":\"if (prevValue === 0) {\\n prevValue = Math.random() * 360;\\n}\\nvar value = prevValue + Math.random() * 20 - 10;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 360) {\\n\\tvalue = 360;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":null,\"decimals\":null,\"usePostProcessing\":null,\"postFuncBody\":null},{\"name\":\"f(x)\",\"type\":\"function\",\"label\":\"centerValue\",\"color\":\"#2196f3\",\"settings\":{},\"_hash\":0.2392660816082064,\"funcBody\":\"var value = prevValue + Math.random() * 7;\\nif (value < 0) {\\n\\tvalue = 0;\\n} else if (value > 30) {\\n\\tvalue = 0;\\n}\\nreturn value;\",\"aggregationType\":null,\"units\":\"m/s\",\"decimals\":1,\"usePostProcessing\":null,\"postFuncBody\":null}],\"alarmFilterConfig\":{\"statusList\":[\"ACTIVE\"]}}],\"timewindow\":{\"realtime\":{\"timewindowMs\":60000}},\"showTitle\":false,\"backgroundColor\":\"rgba(0, 0, 0, 0)\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"padding\":\"0px\",\"settings\":{\"layout\":\"default\",\"centerValueFont\":{\"family\":\"Roboto\",\"size\":24,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"32px\"},\"centerValueColor\":{\"type\":\"constant\",\"color\":\"rgba(0, 0, 0, 0.87)\",\"colorFunction\":\"var temperature = value;\\nif (typeof temperature !== undefined) {\\n var percent = (temperature + 60)/120 * 100;\\n return tinycolor.mix('blue', 'red', percent).toHexString();\\n}\\nreturn 'blue';\"},\"ticksColor\":\"rgba(0, 0, 0, 0.12)\",\"directionalNamesElseDegrees\":true,\"majorTicksFont\":{\"family\":\"Roboto\",\"size\":14,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"20px\"},\"majorTicksColor\":\"rgba(158, 158, 158, 1)\",\"minorTicksFont\":{\"family\":\"Roboto\",\"size\":14,\"sizeUnit\":\"px\",\"style\":\"normal\",\"weight\":\"500\",\"lineHeight\":\"20px\"},\"minorTicksColor\":\"rgba(0, 0, 0, 0.12)\",\"arrowColor\":\"rgba(0, 0, 0, 0.87)\",\"background\":{\"type\":\"color\",\"color\":\"#fff\",\"overlay\":{\"enabled\":false,\"color\":\"rgba(255,255,255,0.72)\",\"blur\":3}}},\"title\":\"Wind speed and direction\",\"dropShadow\":true,\"enableFullscreen\":false,\"titleStyle\":{\"fontSize\":\"16px\",\"fontWeight\":400},\"units\":\"\",\"decimals\":0,\"useDashboardTimewindow\":true,\"showLegend\":false,\"widgetStyle\":{},\"actions\":{\"headerButton\":[]},\"configMode\":\"basic\",\"displayTimewindow\":true,\"margin\":\"0px\",\"borderRadius\":\"0px\",\"widgetCss\":\"\",\"pageSize\":1024,\"noDataDisplayMessage\":\"\",\"showTitleIcon\":false,\"titleTooltip\":\"\",\"titleFont\":{\"size\":16,\"sizeUnit\":\"px\",\"family\":\"Roboto\",\"weight\":\"500\",\"style\":\"normal\",\"lineHeight\":\"24px\"},\"titleIcon\":\"air\",\"iconColor\":\"rgba(0, 0, 0, 0.87)\",\"iconSize\":\"24px\",\"timewindowStyle\":{\"showIcon\":true,\"iconSize\":\"14px\",\"icon\":\"query_builder\",\"iconPosition\":\"left\",\"font\":{\"size\":12,\"sizeUnit\":\"px\",\"family\":null,\"weight\":null,\"style\":null,\"lineHeight\":\"1\"},\"color\":null},\"titleColor\":\"rgba(0, 0, 0, 0.87)\"}" + }, + "externalId": null, + "tags": [ + "wind", + "weather", + "compass", + "degrees" + ] +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 29dcd601a1..a40913a281 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -532,6 +532,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { this.deleteSystemWidgetBundle("html_widgets"); this.deleteSystemWidgetBundle("tables"); this.deleteSystemWidgetBundle("count_widgets"); + this.deleteSystemWidgetBundle("weather_widgets"); installScripts.loadSystemWidgets(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java index 27f4ef918f..f3f3bf5139 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeInfoRepository.java @@ -37,7 +37,7 @@ public interface WidgetTypeInfoRepository extends JpaRepository { - if (dataKey.label === label) { + if (dataKey?.label === label) { i++; label = name + ' ' + i; matches = true; @@ -777,7 +777,7 @@ export function genNextLabel(name: string, datasources: Datasource[]): string { } if (datasource.latestDataKeys) { datasource.latestDataKeys.forEach((dataKey) => { - if (dataKey.label === label) { + if (dataKey?.label === label) { i++; label = name + ' ' + i; matches = true; diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts index 41115904b3..3cc2eddb7d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/basic-widget-config.module.ts @@ -58,6 +58,9 @@ import { import { BatteryLevelBasicConfigComponent } from '@home/components/widget/config/basic/indicator/battery-level-basic-config.component'; +import { + WindSpeedDirectionBasicConfigComponent +} from '@home/components/widget/config/basic/weather/wind-speed-direction-basic-config.component'; @NgModule({ declarations: [ @@ -75,7 +78,8 @@ import { DataKeysPanelComponent, AlarmCountBasicConfigComponent, EntityCountBasicConfigComponent, - BatteryLevelBasicConfigComponent + BatteryLevelBasicConfigComponent, + WindSpeedDirectionBasicConfigComponent ], imports: [ CommonModule, @@ -97,7 +101,8 @@ import { DataKeysPanelComponent, AlarmCountBasicConfigComponent, EntityCountBasicConfigComponent, - BatteryLevelBasicConfigComponent + BatteryLevelBasicConfigComponent, + WindSpeedDirectionBasicConfigComponent ] }) export class BasicWidgetConfigModule { @@ -113,5 +118,6 @@ export const basicWidgetConfigComponentsMap: {[key: string]: Type -
+
{{ 'datakey.timeseries' | translate }} @@ -23,8 +23,8 @@ - - +
@@ -67,8 +67,8 @@ #keyInput [formControl]="keyFormControl" matAutocompleteOrigin - [fxHide]="!!modelValue.type" - [readonly]="!!modelValue.type" + [fxHide]="!!modelValue?.type" + [readonly]="!!modelValue?.type" #origin="matAutocompleteOrigin" [matAutocompleteConnectedTo]="origin" (focusin)="onKeyInputFocus()" @@ -139,7 +139,7 @@ - +
@@ -147,25 +147,28 @@ formControlName="color">
-
+
-
+
+
widget-config.decimals-suffix
-
- +
+
+ +
+ + +
+ +
+ + +
@@ -63,7 +63,7 @@ : ''" formControlName="icon"> - +
@@ -76,7 +76,7 @@ - +
@@ -86,7 +86,7 @@ - +
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts index 5a2a91b56b..4018549863 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/widget-settings-common.module.ts @@ -27,7 +27,10 @@ import { } from '@home/components/widget/lib/settings/common/image-cards-select.component'; import { FontSettingsComponent } from '@home/components/widget/lib/settings/common/font-settings.component'; import { FontSettingsPanelComponent } from '@home/components/widget/lib/settings/common/font-settings-panel.component'; -import { ColorSettingsComponent } from '@home/components/widget/lib/settings/common/color-settings.component'; +import { + ColorSettingsComponent, + ColorSettingsComponentService +} from '@home/components/widget/lib/settings/common/color-settings.component'; import { ColorSettingsPanelComponent } from '@home/components/widget/lib/settings/common/color-settings-panel.component'; @@ -83,6 +86,9 @@ import { LegendConfigComponent, WidgetFontComponent, CountWidgetSettingsComponent + ], + providers: [ + ColorSettingsComponentService ] }) export class WidgetSettingsCommonModule { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html index 1221e5bfc2..7e53d17df1 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html @@ -40,18 +40,18 @@ [autoScale]="batteryLevelWidgetSettingsForm.get('autoScaleValueSize').value" [previewText]="valuePreviewFn"> - +
{{ 'widgets.battery-level.battery-level-color' | translate }}
- +
{{ 'widgets.battery-level.battery-shape-color' | translate }}
- +
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.html index e81519ff3c..1bf2095b10 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/weather/wind-speed-direction-widget-settings.component.html @@ -36,7 +36,7 @@ [initialPreviewStyle]="{ color: windSpeedDirectionWidgetSettingsForm.get('centerValueColor').value?.color }" [previewText]="centerValuePreviewFn"> - +
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 40abb989c1..e755f16420 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5201,7 +5201,8 @@ "value-range": "Value range", "from": "From", "to": "To", - "color-function": "Color function" + "color-function": "Color function", + "copy-color-settings-from": "Copy color settings from" }, "dashboard-state": { "dashboard-state-settings": "Dashboard state settings", From c836c483663d24f10f9d1ef801714014b42ae4d7 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 10 Oct 2023 13:41:45 +0300 Subject: [PATCH 35/36] UI: Implement sections count parameter for battery level widget. --- .../battery-level-basic-config.component.html | 6 +++++ .../battery-level-basic-config.component.ts | 20 +++++++++++--- .../battery-level-widget.component.html | 3 ++- .../battery-level-widget.component.ts | 26 ++++++++++++++++++- .../indicator/battery-level-widget.models.ts | 2 ++ ...ttery-level-widget-settings.component.html | 6 +++++ ...battery-level-widget-settings.component.ts | 21 ++++++++++++--- .../assets/locale/locale.constant-en_US.json | 3 ++- 8 files changed, 78 insertions(+), 9 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.html b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.html index 245ae063a1..1d67d587fe 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.html @@ -40,6 +40,12 @@ {{ batteryLevelLayoutTranslationMap.get(layout) | translate }} +
+
{{ 'widgets.battery-level.sections-count' | translate }}
+ + + +
{{ 'widget-config.title' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.ts index 482a8dd2c3..bb6af9d5ad 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/basic/indicator/battery-level-basic-config.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Injector } from '@angular/core'; +import { Component } from '@angular/core'; import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; @@ -35,6 +35,7 @@ import { formatValue, isUndefined } from '@core/utils'; import { cssSizeToStrSize, resolveCssSize } from '@shared/models/widget-settings.models'; import { batteryLevelDefaultSettings, + BatteryLevelLayout, batteryLevelLayoutImages, batteryLevelLayouts, batteryLevelLayoutTranslations, @@ -67,9 +68,13 @@ export class BatteryLevelBasicConfigComponent extends BasicWidgetConfigComponent valuePreviewFn = this._valuePreviewFn.bind(this); + get sectionsCountEnabled(): boolean { + const layout: BatteryLevelLayout = this.batteryLevelWidgetConfigForm.get('layout').value; + return [BatteryLevelLayout.vertical_divided, BatteryLevelLayout.horizontal_divided].includes(layout); + } + constructor(protected store: Store, protected widgetConfigComponent: WidgetConfigComponent, - private $injector: Injector, private fb: UntypedFormBuilder) { super(store, widgetConfigComponent); } @@ -90,6 +95,7 @@ export class BatteryLevelBasicConfigComponent extends BasicWidgetConfigComponent datasources: [configData.config.datasources, []], layout: [settings.layout, []], + sectionsCount: [settings.sectionsCount, [Validators.min(2), Validators.max(20)]], showTitle: [configData.config.showTitle, []], title: [configData.config.title, []], @@ -136,6 +142,7 @@ export class BatteryLevelBasicConfigComponent extends BasicWidgetConfigComponent this.widgetConfig.config.settings = this.widgetConfig.config.settings || {}; this.widgetConfig.config.settings.layout = config.layout; + this.widgetConfig.config.settings.sectionsCount = config.sectionsCount; this.widgetConfig.config.settings.showValue = config.showValue; this.widgetConfig.config.settings.autoScaleValueSize = config.autoScaleValueSize === true; @@ -155,13 +162,15 @@ export class BatteryLevelBasicConfigComponent extends BasicWidgetConfigComponent } protected validatorTriggers(): string[] { - return ['showTitle', 'showIcon', 'showValue']; + return ['showTitle', 'showIcon', 'showValue', 'layout']; } protected updateValidators(emitEvent: boolean, trigger?: string) { const showTitle: boolean = this.batteryLevelWidgetConfigForm.get('showTitle').value; const showIcon: boolean = this.batteryLevelWidgetConfigForm.get('showIcon').value; const showValue: boolean = this.batteryLevelWidgetConfigForm.get('showValue').value; + const layout: BatteryLevelLayout = this.batteryLevelWidgetConfigForm.get('layout').value; + const divided = [BatteryLevelLayout.vertical_divided, BatteryLevelLayout.horizontal_divided].includes(layout); if (showTitle) { this.batteryLevelWidgetConfigForm.get('title').enable(); @@ -199,6 +208,11 @@ export class BatteryLevelBasicConfigComponent extends BasicWidgetConfigComponent this.batteryLevelWidgetConfigForm.get('valueFont').disable(); this.batteryLevelWidgetConfigForm.get('valueColor').disable(); } + if (divided) { + this.batteryLevelWidgetConfigForm.get('sectionsCount').enable(); + } else { + this.batteryLevelWidgetConfigForm.get('sectionsCount').disable(); + } } private getCardButtons(config: WidgetConfig): string[] { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.html index d5739683b7..cc2fe2cf8f 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.html @@ -22,7 +22,7 @@
-
+
@@ -38,5 +38,6 @@
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.ts index 6e9f8323be..9161764223 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.component.ts @@ -117,7 +117,9 @@ export class BatteryLevelWidgetComponent implements OnInit, OnDestroy, AfterView value: number; - batterySections: boolean[] = [false, false, false, false]; + batterySections: boolean[]; + dividedBorderRadius: string; + dividedGap: string; batteryLevelColor: ColorProcessor; @@ -158,6 +160,26 @@ export class BatteryLevelWidgetComponent implements OnInit, OnDestroy, AfterView this.vertical = [BatteryLevelLayout.vertical_solid, BatteryLevelLayout.vertical_divided].includes(this.layout); this.layoutClass = this.vertical ? 'vertical' : 'horizontal'; this.solid = [BatteryLevelLayout.vertical_solid, BatteryLevelLayout.horizontal_solid].includes(this.layout); + if (!this.solid) { + let sectionsCount = this.settings.sectionsCount; + if (!sectionsCount) { + sectionsCount = 4; + } + sectionsCount = Math.min(Math.max(sectionsCount, 2), 20); + this.batterySections = Array.from(Array(sectionsCount), () => false); + const gap = 1 + (24 - sectionsCount) / 10; + this.dividedGap = `${gap}%`; + const containerAspect = 0.5567; + const sectionHeight = (100 - (gap * (sectionsCount - 1))) / sectionsCount; + const sectionAspect = 100 * containerAspect / sectionHeight; + const rad1 = 8.425 - sectionsCount * 0.32125; + const rad2 = rad1 * sectionAspect; + if (this.vertical) { + this.dividedBorderRadius = `${rad1}% / ${rad2}%`; + } else { + this.dividedBorderRadius = `${rad2}% / ${rad1}%`; + } + } this.showValue = this.settings.showValue; this.autoScaleValueSize = this.showValue && this.settings.autoScaleValueSize; @@ -243,6 +265,8 @@ export class BatteryLevelWidgetComponent implements OnInit, OnDestroy, AfterView const valueLineHeight = ratios.valueLineHeightRaio * boxSize; this.setValueFontSize(valueFontSize, valueLineHeight, boxWidth); } + const fontSize = parseInt(window.getComputedStyle(this.batteryLevelValue.nativeElement).fontSize, 10) || 10; + this.renderer.setStyle(this.batteryLevelValue.nativeElement, 'minWidth', `${Math.min(fontSize*4, boxWidth)}px`); } let height = this.batteryLevelContent.nativeElement.getBoundingClientRect().height; const width = height * verticalBatteryDimensions.shapeAspectRatio; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.models.ts index 8d8724f901..34c00a4b92 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/indicator/battery-level-widget.models.ts @@ -53,6 +53,7 @@ export const batteryLevelLayoutImages = new Map( export interface BatteryLevelWidgetSettings { layout: BatteryLevelLayout; + sectionsCount: number; showValue: boolean; autoScaleValueSize: boolean; valueFont: Font; @@ -64,6 +65,7 @@ export interface BatteryLevelWidgetSettings { export const batteryLevelDefaultSettings: BatteryLevelWidgetSettings = { layout: BatteryLevelLayout.vertical_solid, + sectionsCount: 4, showValue: true, autoScaleValueSize: true, valueFont: { diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html index 7e53d17df1..2b841ca9f8 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.html @@ -28,6 +28,12 @@ {{ batteryLevelLayoutTranslationMap.get(layout) | translate }} +
+
{{ 'widgets.battery-level.sections-count' | translate }}
+ + + +
{{ 'widgets.battery-level.value' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.ts index 627bf00cd2..6c6fecf662 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/indicator/battery-level-widget-settings.component.ts @@ -16,12 +16,12 @@ import { Component, Injector } from '@angular/core'; import { WidgetSettings, WidgetSettingsComponent } from '@shared/models/widget.models'; -import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; +import { UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { formatValue } from '@core/utils'; import { - batteryLevelDefaultSettings, + batteryLevelDefaultSettings, BatteryLevelLayout, batteryLevelLayoutImages, batteryLevelLayouts, batteryLevelLayoutTranslations @@ -43,6 +43,11 @@ export class BatteryLevelWidgetSettingsComponent extends WidgetSettingsComponent valuePreviewFn = this._valuePreviewFn.bind(this); + get sectionsCountEnabled(): boolean { + const layout: BatteryLevelLayout = this.batteryLevelWidgetSettingsForm.get('layout').value; + return [BatteryLevelLayout.vertical_divided, BatteryLevelLayout.horizontal_divided].includes(layout); + } + constructor(protected store: Store, private $injector: Injector, private fb: UntypedFormBuilder) { @@ -60,6 +65,7 @@ export class BatteryLevelWidgetSettingsComponent extends WidgetSettingsComponent protected onSettingsSet(settings: WidgetSettings) { this.batteryLevelWidgetSettingsForm = this.fb.group({ layout: [settings.layout, []], + sectionsCount: [settings.sectionsCount, [Validators.min(2), Validators.max(20)]], showValue: [settings.showValue, []], autoScaleValueSize: [settings.autoScaleValueSize, []], @@ -74,11 +80,13 @@ export class BatteryLevelWidgetSettingsComponent extends WidgetSettingsComponent } protected validatorTriggers(): string[] { - return ['showValue']; + return ['showValue', 'layout']; } protected updateValidators(emitEvent: boolean) { const showValue: boolean = this.batteryLevelWidgetSettingsForm.get('showValue').value; + const layout: BatteryLevelLayout = this.batteryLevelWidgetSettingsForm.get('layout').value; + const divided = [BatteryLevelLayout.vertical_divided, BatteryLevelLayout.horizontal_divided].includes(layout); if (showValue) { this.batteryLevelWidgetSettingsForm.get('autoScaleValueSize').enable(); @@ -90,9 +98,16 @@ export class BatteryLevelWidgetSettingsComponent extends WidgetSettingsComponent this.batteryLevelWidgetSettingsForm.get('valueColor').disable(); } + if (divided) { + this.batteryLevelWidgetSettingsForm.get('sectionsCount').enable(); + } else { + this.batteryLevelWidgetSettingsForm.get('sectionsCount').disable(); + } + this.batteryLevelWidgetSettingsForm.get('autoScaleValueSize').updateValueAndValidity({emitEvent}); this.batteryLevelWidgetSettingsForm.get('valueFont').updateValueAndValidity({emitEvent}); this.batteryLevelWidgetSettingsForm.get('valueColor').updateValueAndValidity({emitEvent}); + this.batteryLevelWidgetSettingsForm.get('sectionsCount').updateValueAndValidity({emitEvent}); } private _valuePreviewFn(): string { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index e755f16420..0fdf48d12c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5057,7 +5057,8 @@ "auto-scale": "Auto scale", "battery-level-color": "Battery level color", "battery-shape-color": "Battery shape color", - "battery-level-card-style": "Battery level card style" + "battery-level-card-style": "Battery level card style", + "sections-count": "Sections count" }, "chart": { "common-settings": "Common settings", From 3ab6b0e2b7a9216a52b5344954d015af2084080d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 10 Oct 2023 17:23:12 +0300 Subject: [PATCH 36/36] UI: Fix data keys chips validation. --- .../widget/config/data-keys.component.ts | 70 ++++++++++++++----- .../widget/widget-config.component.ts | 1 + 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/data-keys.component.ts b/ui-ngx/src/app/modules/home/components/widget/config/data-keys.component.ts index 02060db364..24b2f6345b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/data-keys.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/config/data-keys.component.ts @@ -33,13 +33,13 @@ import { import { AbstractControl, ControlValueAccessor, - FormGroupDirective, + FormGroupDirective, NG_VALIDATORS, NG_VALUE_ACCESSOR, NgForm, UntypedFormBuilder, UntypedFormControl, UntypedFormGroup, - ValidationErrors + ValidationErrors, Validator } from '@angular/forms'; import { Observable, of } from 'rxjs'; import { filter, map, mergeMap, publishReplay, refCount, share, tap } from 'rxjs/operators'; @@ -62,7 +62,7 @@ import { DataKeyConfigDialogComponent, DataKeyConfigDialogData } from '@home/components/widget/config/data-key-config-dialog.component'; -import { deepClone, guid, isDefinedAndNotNull, isUndefined } from '@core/utils'; +import { deepClone, guid, isDefinedAndNotNull, isObject, isUndefined } from '@core/utils'; import { Dashboard } from '@shared/models/dashboard.models'; import { AggregationType } from '@shared/models/time/time.models'; import { DndDropEvent } from 'ngx-drag-drop/lib/dnd-dropzone.directive'; @@ -81,15 +81,20 @@ import { TbPopoverService } from '@shared/components/popover.service'; provide: NG_VALUE_ACCESSOR, useExisting: forwardRef(() => DataKeysComponent), multi: true - } /*, + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => DataKeysComponent), + multi: true, + }, { provide: ErrorStateMatcher, useExisting: DataKeysComponent - } */ + } ], encapsulation: ViewEncapsulation.None }) -export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChanges, ErrorStateMatcher { +export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChanges, ErrorStateMatcher, Validator { public get hideDataKeyLabel(): boolean { return this.datasourceComponent.hideDataKeyLabel; @@ -205,6 +210,9 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange private propagateChange = (v: any) => { }; + private keysRequired = this._keysRequired.bind(this); + private keysValidator = this._keysValidator.bind(this); + constructor(private store: Store, @SkipSelf() private errorStateMatcher: ErrorStateMatcher, private datasourceComponent: DatasourceComponent, @@ -220,12 +228,16 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange } updateValidators() { - this.keysListFormGroup.get('keys').setValidators(this.required ? [this.keysRequired] : []); + if (this.required) { + this.keysListFormGroup.get('keys').addValidators(this.keysRequired); + } else { + this.keysListFormGroup.get('keys').removeValidators(this.keysRequired); + } this.keysListFormGroup.get('keys').updateValueAndValidity(); } - keysRequired(control: AbstractControl): ValidationErrors | null { - const value = control.value; + private _keysRequired(control: AbstractControl): ValidationErrors | null { + const value = this.modelValue; if (value && Array.isArray(value) && value.length) { return null; } else { @@ -233,8 +245,23 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange } } + private _keysValidator(control: AbstractControl): ValidationErrors | null { + const value = this.modelValue; + if (value && Array.isArray(value)) { + if (value.some(v => isObject(v) && (!v.type || !v.name))) { + return { + dataKey: true + }; + } + } + return null; + } + registerOnChange(fn: any): void { this.propagateChange = fn; + if (!this.keysListFormGroup.valid) { + this.propagateChange(this.modelValue); + } } registerOnTouched(fn: any): void { @@ -242,9 +269,12 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange ngOnInit() { this.keysListFormGroup = this.fb.group({ - keys: [null, this.required ? [this.keysRequired] : []], + keys: [null, [this.keysValidator]], key: [null] }); + if (this.required) { + this.keysListFormGroup.get('keys').addValidators(this.keysRequired); + } this.alarmKeys = []; for (const name of Object.keys(alarmFields)) { this.alarmKeys.push({ @@ -335,8 +365,8 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange } else { this.keys = []; } - this.keysListFormGroup.get('keys').setValue(this.keys); this.modelValue = this.keys.length ? [...this.keys] : null; + this.keysListFormGroup.get('keys').setValue(this.keys); if (this.keyInput) { this.keyInput.nativeElement.value = ''; } @@ -374,7 +404,7 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange isErrorState(control: UntypedFormControl | null, form: FormGroupDirective | NgForm | null): boolean { const originalErrorState = this.errorStateMatcher.isErrorState(control, form); - const customErrorState = this.required && (!this.modelValue || !this.modelValue.length); + const customErrorState = this.keysListFormGroup.get('keys').hasError('dataKey'); return originalErrorState || customErrorState; } @@ -395,12 +425,20 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange this.keysListFormGroup.get('keys').setValue(this.keys); } else { this.keys = []; - this.keysListFormGroup.get('keys').setValue(this.keys); this.modelValue = null; + this.keysListFormGroup.get('keys').setValue(this.keys); } this.dirty = true; } + validate(c: UntypedFormControl) { + return (this.keysListFormGroup.get('keys').hasError('dataKey')) ? { + dataKeys: { + valid: false, + }, + } : null; + } + onFocus() { if (this.dirty) { this.keysListFormGroup.get('key').updateValueAndValidity({onlySelf: true, emitEvent: true}); @@ -441,8 +479,8 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange const index = this.keys.indexOf(key); if (index >= 0) { this.keys.splice(index, 1); - this.keysListFormGroup.get('keys').setValue(this.keys); this.modelValue.splice(index, 1); + this.keysListFormGroup.get('keys').setValue(this.keys); if (!this.modelValue.length) { this.modelValue = null; } @@ -468,8 +506,8 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange index = this.keys.length; } moveItemInArray(this.keys, this.dragIndex, index); - this.keysListFormGroup.get('keys').setValue(this.keys); moveItemInArray(this.modelValue, this.dragIndex, index); + this.keysListFormGroup.get('keys').setValue(this.keys); this.dragIndex = -1; this.propagateChange(this.modelValue); } @@ -525,8 +563,8 @@ export class DataKeysComponent implements ControlValueAccessor, OnInit, OnChange }).afterClosed().subscribe((updatedDataKey) => { if (updatedDataKey) { this.keys[index] = updatedDataKey; - this.keysListFormGroup.get('keys').setValue(this.keys); this.modelValue[index] = updatedDataKey; + this.keysListFormGroup.get('keys').setValue(this.keys); this.propagateChange(this.modelValue); } }); diff --git a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts index b4495a2b95..0f569c5ce9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/widget-config.component.ts @@ -435,6 +435,7 @@ export class WidgetConfigComponent extends PageComponent implements OnInit, OnDe this.basicModeComponentChangeSubscription = this.basicModeComponent.widgetConfigChanged.subscribe((data) => { this.modelValue = data; this.propagateChange(this.modelValue); + this.cd.markForCheck(); }); this.cd.markForCheck(); }, 0);