Browse Source

migrated assets used for storing queue statistics to queue_stats table

pull/10145/head
dashevchenko 3 years ago
parent
commit
4223895081
  1. 20
      application/src/main/data/upgrade/3.6.2/schema_update.sql
  2. 13
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  3. 3
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java
  4. 14
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java
  5. 11
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java
  6. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  7. 47
      application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java
  8. 3
      application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java
  9. 20
      application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java
  10. 35
      common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java
  11. 3
      common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java
  12. 43
      common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java
  13. 37
      common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java
  14. 8
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  15. 69
      dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java
  16. 77
      dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java
  17. 30
      dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java
  18. 4
      dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java
  19. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java
  20. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java
  21. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  22. 61
      dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java
  23. 30
      dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java
  24. 9
      dao/src/main/resources/sql/schema-entities.sql

20
application/src/main/data/upgrade/3.6.2/schema_update.sql

@ -28,3 +28,23 @@ ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS queue_name varchar(255);
ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS has_queue_name boolean DEFAULT false;
-- RULE NODE QUEUE UPDATE END
-- QUEUE STATS UPDATE START
CREATE TABLE IF NOT EXISTS queue_stats (
id uuid NOT NULL CONSTRAINT queue_stats_pkey PRIMARY KEY,
created_time bigint NOT NULL,
tenant_id uuid NOT NULL,
queue_name varchar(255) NOT NULL,
service_id varchar(255) NOT NULL,
CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id));
INSERT INTO queue_stats
SELECT id, created_time, tenant_id, split_part(name, '_', 1) AS queue_name, split_part(name, '_', 2) AS service_id
FROM asset
WHERE type = 'TbServiceQueue';
DELETE FROM asset WHERE type='TbServiceQueue';
DELETE FROM asset_profile WHERE name ='TbServiceQueue';
-- QUEUE STATS UPDATE END

13
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -78,6 +78,7 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
@ -96,6 +97,7 @@ import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainType;
@ -123,6 +125,7 @@ import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.queue.QueueStatsService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rpc.RpcService;
@ -307,6 +310,9 @@ public abstract class BaseController {
@Autowired
protected QueueService queueService;
@Autowired
protected QueueStatsService queueStatsService;
@Autowired
protected EntitiesVersionControlService vcService;
@ -600,6 +606,9 @@ public abstract class BaseController {
case QUEUE:
checkQueueId(new QueueId(entityId.getId()), operation);
return;
case QUEUE_STATS:
checkQueueStatsId(new QueueStatsId(entityId.getId()), operation);
return;
default:
checkEntityId(entityId, entitiesService::findEntityByTenantIdAndId, operation);
}
@ -776,6 +785,10 @@ public abstract class BaseController {
return queue;
}
protected QueueStats checkQueueStatsId(QueueStatsId queueStatsId, Operation operation) throws ThingsboardException {
return checkEntityId(queueStatsId, queueStatsService::findQueueStatsById, operation);
}
protected <I extends EntityId> I emptyId(EntityType entityType) {
return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID);
}

3
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java

@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.asset.BaseAssetService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg;
import org.thingsboard.server.gen.edge.v1.DownlinkMsg;
@ -115,7 +114,7 @@ public abstract class AssetEdgeProcessor extends BaseAssetProcessor implements A
case ASSIGNED_TO_CUSTOMER:
case UNASSIGNED_FROM_CUSTOMER:
Asset asset = assetService.findAssetById(edgeEvent.getTenantId(), assetId);
if (asset != null && !BaseAssetService.TB_SERVICE_QUEUE.equals(asset.getType())) {
if (asset != null) {
UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction());
AssetUpdateMsg assetUpdateMsg = ((AssetMsgConstructor)
assetMsgConstructorFactory.getMsgConstructorByEdgeVersion(edgeVersion)).constructAssetUpdatedMsg(msgType, asset);

14
application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@ -22,10 +22,8 @@ import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId;
@ -34,30 +32,18 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
@Service
@AllArgsConstructor
public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService {
private final AssetService assetService;
private final TbAssetProfileCache assetProfileCache;
@Override
public Asset save(Asset asset, User user) throws Exception {
ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = asset.getTenantId();
try {
if (TB_SERVICE_QUEUE.equals(asset.getType())) {
throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
} else if (asset.getAssetProfileId() != null) {
AssetProfile assetProfile = assetProfileCache.get(tenantId, asset.getAssetProfileId());
if (assetProfile != null && TB_SERVICE_QUEUE.equals(assetProfile.getName())) {
throw new ThingsboardException("Unable to save asset with profile " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
}
Asset savedAsset = checkNotNull(assetService.saveAsset(asset));
autoCommit(user, savedAsset.getId());
notificationEntityService.logEntityAction(tenantId, savedAsset.getId(), savedAsset, asset.getCustomerId(),

11
application/src/main/java/org/thingsboard/server/service/entitiy/asset/profile/DefaultTbAssetProfileService.java

@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetProfileId;
import org.thingsboard.server.common.data.id.TenantId;
@ -31,8 +30,6 @@ import org.thingsboard.server.dao.asset.AssetProfileService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
@Service
@TbCoreComponent
@AllArgsConstructor
@ -46,14 +43,6 @@ public class DefaultTbAssetProfileService extends AbstractTbEntityService implem
ActionType actionType = assetProfile.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = assetProfile.getTenantId();
try {
if (TB_SERVICE_QUEUE.equals(assetProfile.getName())) {
throw new ThingsboardException("Unable to save asset profile with name " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS);
} else if (assetProfile.getId() != null) {
AssetProfile foundAssetProfile = assetProfileService.findAssetProfileById(tenantId, assetProfile.getId());
if (foundAssetProfile != null && TB_SERVICE_QUEUE.equals(foundAssetProfile.getName())) {
throw new ThingsboardException("Updating asset profile with name " + TB_SERVICE_QUEUE + " is prohibited!", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
}
AssetProfile savedAssetProfile = checkNotNull(assetProfileService.saveAssetProfile(assetProfile));
autoCommit(user, savedAssetProfile.getId());
tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedAssetProfile.getId(),

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -46,7 +46,8 @@ public enum Resource {
QUEUE(EntityType.QUEUE),
VERSION_CONTROL,
NOTIFICATION(EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_TEMPLATE,
EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE);
EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE),
QUEUE_STATS(EntityType.QUEUE_STATS);
private final Set<EntityType> entityTypes;

47
application/src/main/java/org/thingsboard/server/service/stats/DefaultRuleEngineStatisticsService.java

@ -21,15 +21,15 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.queue.QueueStatsService;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
@ -52,7 +52,6 @@ import java.util.stream.Collectors;
@RequiredArgsConstructor
public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsService {
public static final String TB_SERVICE_QUEUE = "TbServiceQueue";
public static final String RULE_ENGINE_EXCEPTION = "ruleEngineException";
public static final FutureCallback<Integer> CALLBACK = new FutureCallback<Integer>() {
@Override
@ -68,10 +67,10 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
private final TbServiceInfoProvider serviceInfoProvider;
private final TelemetrySubscriptionService tsService;
private final AssetService assetService;
private final QueueStatsService queueStatsService;
private final ApiLimitService apiLimitService;
private final Lock lock = new ReentrantLock();
private final ConcurrentMap<TenantQueueKey, AssetId> tenantQueueAssets = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantQueueKey, QueueStatsId> tenantQueueStats = new ConcurrentHashMap<>();
@Value("${queue.rule-engine.stats.max-error-message-length:4096}")
private int maxErrorMessageLength;
@ -82,7 +81,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
ruleEngineStats.getTenantStats().forEach((id, stats) -> {
try {
TenantId tenantId = TenantId.fromUUID(id);
AssetId serviceAssetId = getServiceAssetId(tenantId, queueName);
QueueStatsId queueStatsId = getQueueStatsId(tenantId, queueName);
if (stats.getTotalMsgCounter().get() > 0) {
List<TsKvEntry> tsList = stats.getCounters().entrySet().stream()
.map(kv -> new BasicTsKvEntry(ts, new LongDataEntry(kv.getKey(), (long) kv.getValue().get())))
@ -90,7 +89,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
if (!tsList.isEmpty()) {
long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getQueueStatsTtlDays);
ttl = TimeUnit.DAYS.toSeconds(ttl);
tsService.saveAndNotifyInternal(tenantId, serviceAssetId, tsList, ttl, CALLBACK);
tsService.saveAndNotifyInternal(tenantId, queueStatsId, tsList, ttl, CALLBACK);
}
}
} catch (Exception e) {
@ -104,7 +103,7 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
TsKvEntry tsKv = new BasicTsKvEntry(e.getTs(), new JsonDataEntry(RULE_ENGINE_EXCEPTION, e.toJsonString(maxErrorMessageLength)));
long ttl = apiLimitService.getLimit(tenantId, DefaultTenantProfileConfiguration::getRuleEngineExceptionsTtlDays);
ttl = TimeUnit.DAYS.toSeconds(ttl);
tsService.saveAndNotifyInternal(tenantId, getServiceAssetId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK);
tsService.saveAndNotifyInternal(tenantId, getQueueStatsId(tenantId, queueName), Collections.singletonList(tsKv), ttl, CALLBACK);
} catch (Exception e2) {
if (!"Asset is referencing to non-existent tenant!".equalsIgnoreCase(e2.getMessage())) {
log.debug("[{}] Failed to store the statistics", tenantId, e2);
@ -113,30 +112,30 @@ public class DefaultRuleEngineStatisticsService implements RuleEngineStatisticsS
});
}
private AssetId getServiceAssetId(TenantId tenantId, String queueName) {
private QueueStatsId getQueueStatsId(TenantId tenantId, String queueName) {
TenantQueueKey key = new TenantQueueKey(tenantId, queueName);
AssetId assetId = tenantQueueAssets.get(key);
if (assetId == null) {
QueueStatsId queueStatsId = tenantQueueStats.get(key);
if (queueStatsId == null) {
lock.lock();
try {
assetId = tenantQueueAssets.get(key);
if (assetId == null) {
Asset asset = assetService.findAssetByTenantIdAndName(tenantId, queueName + "_" + serviceInfoProvider.getServiceId());
if (asset == null) {
asset = new Asset();
asset.setTenantId(tenantId);
asset.setName(queueName + "_" + serviceInfoProvider.getServiceId());
asset.setType(TB_SERVICE_QUEUE);
asset = assetService.saveAsset(asset);
queueStatsId = tenantQueueStats.get(key);
if (queueStatsId == null) {
QueueStats queueStats = queueStatsService.findByTenantIdAndNameAndServiceId(tenantId, queueName , serviceInfoProvider.getServiceId());
if (queueStats == null) {
queueStats = new QueueStats();
queueStats.setTenantId(tenantId);
queueStats.setQueueName(queueName);
queueStats.setServiceId(serviceInfoProvider.getServiceId());
queueStats = queueStatsService.save(tenantId, queueStats);
}
assetId = asset.getId();
tenantQueueAssets.put(key, assetId);
queueStatsId = queueStats.getId();
tenantQueueStats.put(key, queueStatsId);
}
} finally {
lock.unlock();
}
}
return assetId;
return queueStatsId;
}
@Data

3
application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java

@ -50,7 +50,6 @@ import org.thingsboard.server.dao.asset.AssetDao;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService;
import java.util.ArrayList;
import java.util.List;
@ -567,8 +566,6 @@ public class AssetControllerTest extends AbstractControllerTest {
savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.ADDED, cntEntity, cntEntity, cntEntity);
loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE));
assets.sort(idComparator);
loadedAssets.sort(idComparator);

20
application/src/test/java/org/thingsboard/server/controller/BaseQueueControllerTest.java

@ -25,7 +25,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -34,12 +33,13 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.queue.QueueStatsService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.dao.timeseries.TimeseriesDao;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -50,6 +50,7 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingRes
import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@ -66,7 +67,6 @@ import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
@DaoSqlTest
@TestPropertySource(properties = {
@ -81,7 +81,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest {
@SpyBean
private TimeseriesDao timeseriesDao;
@Autowired
private AssetService assetService;
private QueueStatsService queueStatsService;
@Test
public void testQueueWithServiceTypeRE() throws Exception {
@ -176,16 +176,16 @@ public class BaseQueueControllerTest extends AbstractControllerTest {
});
ruleEngineStatisticsService.reportQueueStats(System.currentTimeMillis(), testStats);
Asset serviceAsset = assetService.findAssetsByTenantIdAndType(tenantId, TB_SERVICE_QUEUE, new PageLink(100)).getData()
.stream().filter(asset -> asset.getName().startsWith(queue.getName()))
.findFirst().get();
List<QueueStats> queueStatsList = queueStatsService.findByTenantId(tenantId);
assertThat(queueStatsList).hasSize(1);
QueueStats queueStats = queueStatsList.get(0);
ArgumentCaptor<Long> ttlCaptor = ArgumentCaptor.forClass(Long.class);
verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> {
verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> {
return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.SUCCESSFUL_MSGS) &&
tsKvEntry.getLongValue().get().equals(5L);
}), ttlCaptor.capture());
verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> {
verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> {
return tsKvEntry.getKey().equals(TbRuleEngineConsumerStats.FAILED_MSGS) &&
tsKvEntry.getLongValue().get().equals(5L);
}), ttlCaptor.capture());
@ -193,7 +193,7 @@ public class BaseQueueControllerTest extends AbstractControllerTest {
assertThat(usedTtl).isEqualTo(TimeUnit.DAYS.toSeconds(queueStatsTtlDays));
});
verify(timeseriesDao).save(eq(tenantId), eq(serviceAsset.getId()), argThat(tsKvEntry -> {
verify(timeseriesDao).save(eq(tenantId), eq(queueStats.getId()), argThat(tsKvEntry -> {
return tsKvEntry.getKey().equals(DefaultRuleEngineStatisticsService.RULE_ENGINE_EXCEPTION) &&
tsKvEntry.getJsonValue().get().equals(ruleEngineException.toJsonString(0));
}), ttlCaptor.capture());

35
common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java

@ -0,0 +1,35 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.queue;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.dao.entity.EntityDaoService;
import java.util.List;
public interface QueueStatsService extends EntityDaoService {
QueueStats save(TenantId tenantId, QueueStats queueStats);
QueueStats findQueueStatsById(TenantId tenantId, QueueStatsId queueStatsId);
QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueStatsName, String serviceId);
List<QueueStats> findByTenantId(TenantId tenantId);
}

3
common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@ -58,7 +58,8 @@ public enum EntityType {
NOTIFICATION_TEMPLATE (30),
NOTIFICATION_REQUEST (31),
NOTIFICATION (32),
NOTIFICATION_RULE (33);
NOTIFICATION_RULE (33),
QUEUE_STATS(34);
@Getter
private final int protoNumber; // Corresponds to EntityTypeProto

43
common/data/src/main/java/org/thingsboard/server/common/data/id/QueueStatsId.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.id;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.annotations.ApiModelProperty;
import org.thingsboard.server.common.data.EntityType;
import java.util.UUID;
public class QueueStatsId extends UUIDBased implements EntityId {
private static final long serialVersionUID = 1L;
@JsonCreator
public QueueStatsId(@JsonProperty("id") UUID id) {
super(id);
}
public static QueueStatsId fromString(String queueId) {
return new QueueStatsId(UUID.fromString(queueId));
}
@ApiModelProperty(position = 2, required = true, value = "string", example = "QUEUE_STATS", allowableValues = "QUEUE_STATS")
@Override
public EntityType getEntityType() {
return EntityType.QUEUE_STATS;
}
}

37
common/data/src/main/java/org/thingsboard/server/common/data/queue/QueueStats.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
import lombok.Data;
import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.QueueStatsId;
@Data
public class QueueStats extends BaseDataWithAdditionalInfo<QueueStatsId> implements HasTenantId {
private TenantId tenantId;
private String queueName;
private String serviceId;
public QueueStats() {
}
public QueueStats(QueueStatsId id) {
super(id);
}
}

8
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@ -610,6 +610,14 @@ public class ModelConstants {
public static final String QUEUE_TABLE_NAME = "queue";
public static final String QUEUE_ADDITIONAL_INFO_PROPERTY = ADDITIONAL_INFO_PROPERTY;
/**
* Tenant queue stats constants.
*/
public static final String QUEUE_STATS_TABLE_NAME = "queue_stats";
public static final String QUEUE_STATS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY;
public static final String QUEUE_STATS_QUEUE_NAME_PROPERTY = "queue_name";
public static final String QUEUE_STATS_SERVICE_ID_PROPERTY = "service_id";
/**
* Notification constants
*/

69
dao/src/main/java/org/thingsboard/server/dao/model/sql/QueueStatsEntity.java

@ -0,0 +1,69 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.BaseSqlEntity;
import org.thingsboard.server.dao.model.ModelConstants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.util.UUID;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = ModelConstants.QUEUE_STATS_TABLE_NAME)
public class QueueStatsEntity extends BaseSqlEntity<QueueStats> {
@Column(name = ModelConstants.QUEUE_STATS_TENANT_ID_PROPERTY)
private UUID tenantId;
@Column(name = ModelConstants.QUEUE_STATS_QUEUE_NAME_PROPERTY)
private String queueName;
@Column(name = ModelConstants.QUEUE_STATS_SERVICE_ID_PROPERTY)
private String serviceId;
public QueueStatsEntity() {
}
public QueueStatsEntity(QueueStats queueStats) {
if (queueStats.getId() != null) {
this.setId(queueStats.getId().getId());
}
this.setCreatedTime(queueStats.getCreatedTime());
this.tenantId = DaoUtil.getId(queueStats.getTenantId());
this.queueName = queueStats.getQueueName();
this.serviceId = queueStats.getServiceId();
}
@Override
public QueueStats toData() {
QueueStats queueStats = new QueueStats(new QueueStatsId(getUuid()));
queueStats.setCreatedTime(createdTime);
queueStats.setTenantId(new TenantId(tenantId));
queueStats.setQueueName(queueName);
queueStats.setServiceId(serviceId);
return queueStats;
}
}

77
dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java

@ -0,0 +1,77 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.queue;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.QueueStatsId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.dao.entity.AbstractEntityService;
import java.util.List;
import java.util.Optional;
import static org.thingsboard.server.dao.service.Validator.validateId;
@Service("QueueStatsDaoService")
@Slf4j
@RequiredArgsConstructor
public class BaseQueueStatsService extends AbstractEntityService implements QueueStatsService {
@Autowired
private QueueStatsDao queueStatsDao;
@Override
public QueueStats save(TenantId tenantId, QueueStats queueStats) {
return queueStatsDao.save(tenantId, queueStats);
}
@Override
public QueueStats findQueueStatsById(TenantId tenantId, QueueStatsId queueStatsId) {
log.trace("Executing findQueueStatsById [{}]", queueStatsId);
validateId(queueStatsId, "Incorrect queueStatsId " + queueStatsId);
return queueStatsDao.findById(tenantId, queueStatsId.getId());
}
@Override
public QueueStats findByTenantIdAndNameAndServiceId(TenantId tenantId, String queueName, String serviceId) {
log.trace("Executing findByTenantIdAndNameAndServiceId, tenantId: [{}], queueName: [{}], serviceId: [{}]", tenantId, queueName, serviceId);
return queueStatsDao.findByTenantIdQueueNameAndServiceId(tenantId, queueName, serviceId);
}
@Override
public List<QueueStats> findByTenantId(TenantId tenantId) {
log.trace("Executing findByTenantId, tenantId: [{}]", tenantId);
return queueStatsDao.findByTenantId(tenantId);
}
@Override
public Optional<HasId<?>> findEntity(TenantId tenantId, EntityId entityId) {
return Optional.ofNullable(findQueueStatsById(tenantId, new QueueStatsId(entityId.getId())));
}
@Override
public EntityType getEntityType() {
return null;
}
}

30
dao/src/main/java/org/thingsboard/server/dao/queue/QueueStatsDao.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.queue;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.dao.Dao;
import java.util.List;
public interface QueueStatsDao extends Dao<QueueStats> {
QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String name, String serviceId);
List<QueueStats> findByTenantId(TenantId tenantId);
}

4
dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java

@ -47,9 +47,7 @@ public class AssetDataValidator extends DataValidator<Asset> {
@Override
protected void validateCreate(TenantId tenantId, Asset asset) {
if (!BaseAssetService.TB_SERVICE_QUEUE.equals(asset.getType())) {
validateNumberOfEntitiesPerTenant(tenantId, EntityType.ASSET);
}
validateNumberOfEntitiesPerTenant(tenantId, EntityType.ASSET);
}
@Override

2
dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java

@ -189,7 +189,7 @@ public interface AssetRepository extends JpaRepository<AssetEntity, UUID>, Expor
@Param("searchText") String searchText,
Pageable pageable);
Long countByTenantIdAndTypeIsNot(UUID tenantId, String type);
Long countByTenantId(UUID tenantId);
@Query("SELECT externalId FROM AssetEntity WHERE id = :id")
UUID getExternalIdById(@Param("id") UUID id);

3
dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java

@ -43,7 +43,6 @@ import java.util.Optional;
import java.util.UUID;
import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto;
import static org.thingsboard.server.dao.asset.BaseAssetService.TB_SERVICE_QUEUE;
/**
* Created by Valerii Sosliuk on 5/19/2017.
@ -244,7 +243,7 @@ public class JpaAssetDao extends JpaAbstractDao<AssetEntity, Asset> implements A
@Override
public Long countByTenantId(TenantId tenantId) {
return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE);
return assetRepository.countByTenantId(tenantId.getId());
}
@Override

2
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@ -244,6 +244,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityTableMap.put(EntityType.DEVICE_PROFILE, "device_profile");
entityTableMap.put(EntityType.ASSET_PROFILE, "asset_profile");
entityTableMap.put(EntityType.TENANT_PROFILE, "tenant_profile");
entityTableMap.put(EntityType.QUEUE_STATS, "queue_stats");
entityNameColumns.put(EntityType.DEVICE, "name");
entityNameColumns.put(EntityType.CUSTOMER, "title");
@ -262,6 +263,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
entityNameColumns.put(EntityType.TB_RESOURCE, "search_text");
entityNameColumns.put(EntityType.EDGE, "name");
entityNameColumns.put(EntityType.QUEUE, "name");
entityNameColumns.put(EntityType.QUEUE_STATS, "queue_name");
}
public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{

61
dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.queue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.QueueStats;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.model.sql.QueueStatsEntity;
import org.thingsboard.server.dao.queue.QueueStatsDao;
import org.thingsboard.server.dao.sql.JpaAbstractDao;
import org.thingsboard.server.dao.util.SqlDao;
import java.util.List;
import java.util.UUID;
@Slf4j
@Component
@SqlDao
public class JpaQueueStatsDao extends JpaAbstractDao<QueueStatsEntity, QueueStats> implements QueueStatsDao {
@Autowired
private QueueStatsRepository queueStatsRepository;
@Override
protected Class<QueueStatsEntity> getEntityClass() {
return QueueStatsEntity.class;
}
@Override
protected JpaRepository<QueueStatsEntity, UUID> getRepository() {
return queueStatsRepository;
}
@Override
public QueueStats findByTenantIdQueueNameAndServiceId(TenantId tenantId, String name, String serviceId) {
return DaoUtil.getData(queueStatsRepository.findByTenantIdAndQueueNameAndServiceId(tenantId.getId(), name, serviceId));
}
@Override
public List<QueueStats> findByTenantId(TenantId tenantId) {
return DaoUtil.convertDataList(queueStatsRepository.findByTenantId(tenantId.getId()));
}
}

30
dao/src/main/java/org/thingsboard/server/dao/sql/queue/QueueStatsRepository.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.queue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.thingsboard.server.dao.model.sql.QueueStatsEntity;
import java.util.List;
import java.util.UUID;
public interface QueueStatsRepository extends JpaRepository<QueueStatsEntity, UUID> {
QueueStatsEntity findByTenantIdAndQueueNameAndServiceId(UUID tenantId, String name, String serviceId);
List<QueueStatsEntity> findByTenantId(UUID tenantId);
}

9
dao/src/main/resources/sql/schema-entities.sql

@ -884,3 +884,12 @@ CREATE TABLE IF NOT EXISTS alarm_types (
CONSTRAINT tenant_id_type_unq_key UNIQUE (tenant_id, type),
CONSTRAINT fk_entity_tenant_id FOREIGN KEY (tenant_id) REFERENCES tenant(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS queue_stats (
id uuid NOT NULL CONSTRAINT queue_stats_pkey PRIMARY KEY,
created_time bigint NOT NULL,
tenant_id uuid NOT NULL,
queue_name varchar(255) NOT NULL,
service_id varchar(255) NOT NULL,
CONSTRAINT queue_stats_name_unq_key UNIQUE (tenant_id, queue_name, service_id)
);
Loading…
Cancel
Save