From b13c9d7368eb0c65355b8ddae37f219704f372dd Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sat, 11 Mar 2023 21:29:39 +0100 Subject: [PATCH 01/16] added curent tb version to the UpdateMessage --- .../server/service/update/DefaultUpdateService.java | 5 +++-- .../org/thingsboard/server/common/data/UpdateMessage.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java index ac3bfc497a..7dee304200 100644 --- a/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java @@ -66,7 +66,7 @@ public class DefaultUpdateService implements UpdateService { @PostConstruct private void init() { - updateMessage = new UpdateMessage("", false); + updateMessage = new UpdateMessage("", false, ""); if (updatesEnabled) { try { platform = System.getProperty("platform", "unknown"); @@ -124,7 +124,8 @@ public class DefaultUpdateService implements UpdateService { JsonNode response = restClient.postForObject(UPDATE_SERVER_BASE_URL+"/api/thingsboard/updates", request, JsonNode.class); updateMessage = new UpdateMessage( response.get("message").asText(), - response.get("updateAvailable").asBoolean() + response.get("updateAvailable").asBoolean(), + version ); } catch (Exception e) { log.trace(e.getMessage()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/UpdateMessage.java b/common/data/src/main/java/org/thingsboard/server/common/data/UpdateMessage.java index 203bd578ce..b5ecdcbf79 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/UpdateMessage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/UpdateMessage.java @@ -25,7 +25,8 @@ public class UpdateMessage { @ApiModelProperty(position = 1, value = "The message about new platform update available.") private final String message; - @ApiModelProperty(position = 1, value = "'True' if new platform update is available.") + @ApiModelProperty(position = 2, value = "'True' if new platform update is available.") private final boolean isUpdateAvailable; - + @ApiModelProperty(position = 3, value = "Current ThingsBoard version.") + private final String currentVersion; } From b936d4841943d4d0b7ea5e72791b53201f7b5bd2 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Mar 2023 10:57:04 +0100 Subject: [PATCH 02/16] implemented countEntitiesByTypes for SysAdmin --- .../controller/EntityQueryController.java | 15 +++++ .../query/DefaultEntityQueryService.java | 5 ++ .../service/query/EntityQueryService.java | 6 ++ .../server/dao/entity/EntityService.java | 5 ++ .../server/dao/entity/BaseEntityService.java | 11 ++++ .../server/dao/entity/EntityQueryDao.java | 6 ++ .../query/DefaultEntityQueryRepository.java | 58 ++++++++++++++++++- .../dao/sql/query/EntityQueryRepository.java | 6 ++ .../dao/sql/query/JpaEntityQueryDao.java | 9 +++ 9 files changed, 120 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 1d429e9157..f64dea7b8d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -27,6 +27,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -41,6 +42,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.permission.Operation; +import java.util.List; +import java.util.Map; + import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION; @@ -66,6 +70,17 @@ public class EntityQueryController extends BaseController { return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); } + @ApiOperation(value = "Count Entities by Entity Types") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @RequestMapping(value = "/entitiesTypes/count", method = RequestMethod.POST) + @ResponseBody + public Map countEntitiesByQuery( + @ApiParam(value = "A JSON value representing the entity types array.") + @RequestBody List entityTypes) throws ThingsboardException { + checkNotNull(entityTypes); + return this.entityQueryService.countEntitiesByTypes(getCurrentUser(), entityTypes); + } + @ApiOperation(value = "Find Entity Data by Query", notes = ENTITY_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index 5a9d0d9edf..154f64202a 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -102,6 +102,11 @@ public class DefaultEntityQueryService implements EntityQueryService { return entityService.countEntitiesByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query); } + @Override + public Map countEntitiesByTypes(SecurityUser securityUser, List entityTypes) { + return entityService.countEntitiesByTypes(securityUser.getTenantId(), securityUser.getCustomerId(), entityTypes); + } + @Override public PageData findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query) { if (query.getKeyFilters() != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java index b03fb02b33..8a37eadb70 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.query; import org.springframework.http.ResponseEntity; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; @@ -26,10 +27,15 @@ import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.List; +import java.util.Map; + public interface EntityQueryService { long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query); + Map countEntitiesByTypes(SecurityUser securityUser, List entityTypes); + PageData findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query); PageData findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index f2317efa05..23e33b26e5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.entity; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -24,6 +25,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.List; +import java.util.Map; import java.util.Optional; public interface EntityService { @@ -38,5 +41,7 @@ public interface EntityService { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); + Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); + PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index cbce9e48a1..f4ce434635 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; import org.thingsboard.server.common.data.HasLabel; @@ -38,6 +39,8 @@ import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.dao.exception.IncorrectParameterException; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -71,6 +74,14 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } + @Override + public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { + log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], entityTypes [{}]", tenantId, customerId, entityTypes); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(customerId, INCORRECT_CUSTOMER_ID + customerId); + return this.entityQueryDao.countEntitiesByTypes(tenantId, customerId, entityTypes); + } + @Override public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { log.trace("Executing findEntityDataByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java index 9fafe2e0f0..bcdde392dc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.entity; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -22,10 +23,15 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.List; +import java.util.Map; + public interface EntityQueryDao { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query); + Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 821dbcc011..c4bf057efd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -53,6 +53,7 @@ import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -388,6 +389,61 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return findEntityDataByQuery(tenantId, customerId, query, false); } + @Override + public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { + int size = entityTypes.size(); + + QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, null, true)); + ctx.append("select "); + + for (int i = 0; i < size; i++) { + ctx.append("(select count(*) from "); + ctx.append(getTableName(entityTypes.get(i))); + ctx.append(")"); + if (i < size - 1) { + ctx.append(", "); + } + } + + return transactionTemplate.execute(status -> { + long startTs = System.currentTimeMillis(); + try { + List counts = jdbcTemplate.query(ctx.getQuery(), rs -> { + List result = new ArrayList<>(); + if (rs.next()) { + for (int i = 1; i <= size; i++) { + result.add(rs.getLong(i)); + } + } + return result; + }); + + Map result = new HashMap<>(size); + for (int i = 0; i < size; i++) { + result.put(entityTypes.get(i), counts.get(i)); + } + return result; + } finally { + queryLog.logQuery(ctx, ctx.getQuery(), System.currentTimeMillis() - startTs); + } + }); + } + + private String getTableName(EntityType entityType) { + switch (entityType) { + case TENANT: + case TENANT_PROFILE: + case CUSTOMER: + case DEVICE: + case ASSET: + return entityType.name().toLowerCase(); + case USER: + return "tb_user"; + default: + throw new IllegalArgumentException("Not supported entity type: " + entityType + "!"); + } + } + public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); @@ -514,7 +570,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } private String buildPermissionQuery(QueryContext ctx, EntityFilter entityFilter) { - if(ctx.isIgnorePermissionCheck()){ + if (ctx.isIgnorePermissionCheck()) { return "1=1"; } switch (entityFilter.getType()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java index 7d78a34981..5fd91057ce 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.query; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -22,6 +23,9 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.List; +import java.util.Map; + public interface EntityQueryRepository { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); @@ -30,4 +34,6 @@ public interface EntityQueryRepository { PageData findEntityDataByQueryInternal(EntityDataQuery query); + Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java index 33d9426bdf..3929c98eec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.query; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -25,6 +26,9 @@ import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.dao.entity.EntityQueryDao; +import java.util.List; +import java.util.Map; + @Component public class JpaEntityQueryDao implements EntityQueryDao { @@ -40,4 +44,9 @@ public class JpaEntityQueryDao implements EntityQueryDao { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { return entityQueryRepository.findEntityDataByQuery(tenantId, customerId, query); } + + @Override + public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { + return entityQueryRepository.countEntitiesByTypes(tenantId, customerId, entityTypes); + } } From bb2ea9105116cf6f3e2b6584e97c67dc25854999 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Mar 2023 11:00:33 +0100 Subject: [PATCH 03/16] added countEntitiesByTypes to the RestClient --- .../java/org/thingsboard/rest/client/RestClient.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index c2c0a6d794..7eee546632 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -34,7 +34,6 @@ import org.springframework.http.client.ClientHttpResponse; import org.springframework.http.client.support.HttpRequestWrapper; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import org.thingsboard.server.common.data.StringUtils; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -58,6 +57,7 @@ import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -1584,6 +1584,13 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { return restTemplate.postForObject(baseURL + "/api/entitiesQuery/count", query, Long.class); } + public Map countEntitiesByTypes(List entityTypes) { + return restTemplate.exchange( + baseURL + "/api/entitiesTypes/count", + HttpMethod.POST, new HttpEntity<>(entityTypes), new ParameterizedTypeReference>() { + }).getBody(); + } + public PageData findEntityDataByQuery(EntityDataQuery query) { return restTemplate.exchange( baseURL + "/api/entitiesQuery/find", From 7cd949edb27764ef9c0c1066b691990d20049206 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Mar 2023 20:05:49 +0100 Subject: [PATCH 04/16] implemented system info --- .../server/controller/AdminController.java | 73 ++++++++------- .../system/DefaultSystemInfoService.java | 91 +++++++++++++++++++ .../service/system/SystemInfoService.java | 22 +++++ .../server/common/data/SystemInfo.java | 35 +++++++ .../queue/discovery/HashPartitionService.java | 2 + .../queue/discovery/PartitionService.java | 2 + 6 files changed, 191 insertions(+), 34 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index d2e1123d04..c3078e7f83 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -21,29 +21,38 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import org.springframework.beans.factory.annotation.Autowired; +import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettingsInfo; -import org.thingsboard.server.common.data.security.model.JwtSettings; -import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; @@ -51,43 +60,30 @@ import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; +import org.thingsboard.server.service.system.SystemInfoService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @RequestMapping("/api/admin") +@RequiredArgsConstructor public class AdminController extends BaseController { - @Autowired - private MailService mailService; - - @Autowired - private SmsService smsService; - - @Autowired - private AdminSettingsService adminSettingsService; - - @Autowired - private SystemSecurityService systemSecurityService; - + private final MailService mailService; + private final SmsService smsService; + private final AdminSettingsService adminSettingsService; + private final SystemSecurityService systemSecurityService; @Lazy - @Autowired - private JwtSettingsService jwtSettingsService; - + private final JwtSettingsService jwtSettingsService; @Lazy - @Autowired - private JwtTokenFactory tokenFactory; - - @Autowired - private EntitiesVersionControlService versionControlService; - - @Autowired - private TbAutoCommitSettingsService autoCommitSettingsService; - - @Autowired - private UpdateService updateService; + private final JwtTokenFactory tokenFactory; + private final EntitiesVersionControlService versionControlService; + private final TbAutoCommitSettingsService autoCommitSettingsService; + private final UpdateService updateService; + private final SystemInfoService systemInfoService; @ApiOperation(value = "Get the Administration Settings object using key (getAdminSettings)", notes = "Get the Administration Settings object using specified string key. Referencing non-existing key will cause an error." + SYSTEM_AUTHORITY_PARAGRAPH) @@ -109,7 +105,6 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Get the Administration Settings object using key (getAdminSettings)", notes = "Creates or Updates the Administration Settings. Platform generates random Administration Settings Id during settings creation. " + "The Administration Settings Id will be present in the response. Specify the Administration Settings Id when you would like to update the Administration Settings. " + @@ -318,7 +313,6 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check repository access (checkRepositoryAccess)", notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -399,4 +393,15 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get system info (getSystemInfo)", + notes = "Get main information about system. " + + SYSTEM_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/systemInfo", method = RequestMethod.GET) + @ResponseBody + public SystemInfo getSystemInfo() throws ThingsboardException { + return systemInfoService.getSystemInfo(); + + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java new file mode 100644 index 0000000000..3e9b778a5d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.system; + +import com.google.protobuf.ProtocolStringList; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.SystemInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@TbCoreComponent +@Service +@RequiredArgsConstructor +public class DefaultSystemInfoService implements SystemInfoService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + + @Value("${zk.enabled:false}") + private boolean zkEnabled; + + @Override + @SneakyThrows + public SystemInfo getSystemInfo() { + SystemInfo systemInfo = new SystemInfo(); + + TransportProtos.ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfo(); + List currentOtherServices = partitionService.getCurrentOtherServices(); + + if (zkEnabled) { + Map serviceInfos = new HashMap<>(); + addServiceInfo(serviceInfos, serviceInfo); + currentOtherServices.forEach(otherInfo -> addServiceInfo(serviceInfos, otherInfo)); + systemInfo.setServiceInfos(serviceInfos); + } else { + systemInfo.setMonolith(true); + systemInfo.setMemUsage(getMemoryUsage()); + + systemInfo.setCpuUsage((int) (getCpuUsage() * 100) / 100.0); + systemInfo.setFreeDiscSpace(getFreeDiscSpace()); + } + + return systemInfo; + } + + private void addServiceInfo(Map serviceInfos, TransportProtos.ServiceInfo serviceInfo) { + ProtocolStringList serviceTypes = serviceInfo.getServiceTypesList(); + serviceInfos.put(serviceInfo.getServiceId(), serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0)); + } + + private long getMemoryUsage() { + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + return memoryMXBean.getHeapMemoryUsage().getUsed(); + } + + private double getCpuUsage() { + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + return osBean.getSystemLoadAverage(); + } + + private long getFreeDiscSpace() { + File file = new File("/"); + return file.getFreeSpace(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java new file mode 100644 index 0000000000..3b48d809b2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.system; + +import org.thingsboard.server.common.data.SystemInfo; + +public interface SystemInfoService { + SystemInfo getSystemInfo(); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java new file mode 100644 index 0000000000..f1aa8b4abe --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + +@Data +public class SystemInfo { + @ApiModelProperty(position = 1, value = "Is monolith.") + private boolean isMonolith; + @ApiModelProperty(position = 2, value = "CPU usage.") + private Double cpuUsage; + @ApiModelProperty(position = 3, value = "Memory usage.") + private Long memUsage; + @ApiModelProperty(position = 4, value = "Free disc space.") + private Long freeDiscSpace; + @ApiModelProperty(position = 5, value = "Json object with info about services.") + private Map serviceInfos; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 3fc1626f52..6888b9ad8c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -76,6 +77,7 @@ public class HashPartitionService implements PartitionService { private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); private Map> tbTransportServicesByType = new HashMap<>(); + @Getter private List currentOtherServices; private HashFunction hashFunction; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index f19375984e..3806c59a68 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -63,4 +63,6 @@ public interface PartitionService { void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg); void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); + + List getCurrentOtherServices(); } From 0be3c0b5bbf194393e1c883eb3c187c93a915ee9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 14 Mar 2023 16:26:55 +0100 Subject: [PATCH 05/16] removed countEntitiesByTypes and added ability to use Entity count query for sysadmin --- .../server/controller/AdminController.java | 1 - .../controller/EntityQueryController.java | 17 +---- .../query/DefaultEntityQueryService.java | 7 +- .../service/query/EntityQueryService.java | 6 -- .../BaseEntityQueryControllerTest.java | 52 +++++++++++++- .../server/dao/entity/EntityService.java | 7 +- .../server/dao/entity/BaseEntityService.java | 13 +--- .../server/dao/entity/EntityQueryDao.java | 6 -- .../query/DefaultEntityQueryRepository.java | 68 +++---------------- .../dao/sql/query/EntityQueryRepository.java | 6 -- .../dao/sql/query/JpaEntityQueryDao.java | 8 --- .../thingsboard/rest/client/RestClient.java | 7 -- 12 files changed, 64 insertions(+), 134 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index c3078e7f83..a8eee51f17 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -401,7 +401,6 @@ public class AdminController extends BaseController { @ResponseBody public SystemInfo getSystemInfo() throws ThingsboardException { return systemInfoService.getSystemInfo(); - } } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index f64dea7b8d..3fdc1bc236 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -27,7 +27,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; @@ -42,9 +41,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.permission.Operation; -import java.util.List; -import java.util.Map; - import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION; @@ -60,7 +56,7 @@ public class EntityQueryController extends BaseController { private static final int MAX_PAGE_SIZE = 100; @ApiOperation(value = "Count Entities by Query", notes = ENTITY_COUNT_QUERY_DESCRIPTION) - @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST) @ResponseBody public long countEntitiesByQuery( @@ -70,17 +66,6 @@ public class EntityQueryController extends BaseController { return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); } - @ApiOperation(value = "Count Entities by Entity Types") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") - @RequestMapping(value = "/entitiesTypes/count", method = RequestMethod.POST) - @ResponseBody - public Map countEntitiesByQuery( - @ApiParam(value = "A JSON value representing the entity types array.") - @RequestBody List entityTypes) throws ThingsboardException { - checkNotNull(entityTypes); - return this.entityQueryService.countEntitiesByTypes(getCurrentUser(), entityTypes); - } - @ApiOperation(value = "Find Entity Data by Query", notes = ENTITY_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index 154f64202a..ba5b29c11b 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -29,6 +29,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -55,7 +56,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.security.AccessValidator; @@ -102,11 +102,6 @@ public class DefaultEntityQueryService implements EntityQueryService { return entityService.countEntitiesByQuery(securityUser.getTenantId(), securityUser.getCustomerId(), query); } - @Override - public Map countEntitiesByTypes(SecurityUser securityUser, List entityTypes) { - return entityService.countEntitiesByTypes(securityUser.getTenantId(), securityUser.getCustomerId(), entityTypes); - } - @Override public PageData findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query) { if (query.getKeyFilters() != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java index 8a37eadb70..b03fb02b33 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.query; import org.springframework.http.ResponseEntity; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; @@ -27,15 +26,10 @@ import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.List; -import java.util.Map; - public interface EntityQueryService { long countEntitiesByQuery(SecurityUser securityUser, EntityCountQuery query); - Map countEntitiesByTypes(SecurityUser securityUser, List entityTypes); - PageData findEntityDataByQuery(SecurityUser securityUser, EntityDataQuery query); PageData findAlarmDataByQuery(SecurityUser securityUser, AlarmDataQuery query); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java index aeebc7bfee..f29827fbe2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java @@ -92,7 +92,7 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe } @Test - public void testCountEntitiesByQuery() throws Exception { + public void testTenantCountEntitiesByQuery() throws Exception { List devices = new ArrayList<>(); for (int i = 0; i < 97; i++) { Device device = new Device(); @@ -139,6 +139,56 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe Assert.assertEquals(97, count2.longValue()); } + @Test + public void testSysAdminCountEntitiesByQuery() throws Exception { + List devices = new ArrayList<>(); + for (int i = 0; i < 97; i++) { + Device device = new Device(); + device.setName("Device" + i); + device.setType("default"); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + devices.add(doPost("/api/device", device, Device.class)); + Thread.sleep(1); + } + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceType("default"); + filter.setDeviceNameFilter(""); + + loginSysAdmin(); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + + Long count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + Assert.assertEquals(97, count.longValue()); + + filter.setDeviceType("unknown"); + count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + Assert.assertEquals(0, count.longValue()); + + filter.setDeviceType("default"); + filter.setDeviceNameFilter("Device1"); + + count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + Assert.assertEquals(11, count.longValue()); + + EntityListFilter entityListFilter = new EntityListFilter(); + entityListFilter.setEntityType(EntityType.DEVICE); + entityListFilter.setEntityList(devices.stream().map(Device::getId).map(DeviceId::toString).collect(Collectors.toList())); + + countQuery = new EntityCountQuery(entityListFilter); + + count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class); + Assert.assertEquals(97, count.longValue()); + + EntityTypeFilter filter2 = new EntityTypeFilter(); + filter2.setEntityType(EntityType.DEVICE); + + EntityCountQuery countQuery2 = new EntityCountQuery(filter2); + + Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class); + Assert.assertEquals(97, count2.longValue()); + } + @Test public void testSimpleFindEntityDataByQuery() throws Exception { List devices = new ArrayList<>(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 23e33b26e5..3983494d70 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -15,18 +15,15 @@ */ package org.thingsboard.server.dao.entity; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; -import java.util.List; -import java.util.Map; import java.util.Optional; public interface EntityService { @@ -41,7 +38,5 @@ public interface EntityService { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); - Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); - PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index f4ce434635..a7c918f9ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -19,17 +19,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; import org.thingsboard.server.common.data.HasLabel; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTitle; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -39,8 +38,6 @@ import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.dao.exception.IncorrectParameterException; -import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -74,14 +71,6 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } - @Override - public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { - log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], entityTypes [{}]", tenantId, customerId, entityTypes); - validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - validateId(customerId, INCORRECT_CUSTOMER_ID + customerId); - return this.entityQueryDao.countEntitiesByTypes(tenantId, customerId, entityTypes); - } - @Override public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { log.trace("Executing findEntityDataByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java index bcdde392dc..9fafe2e0f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityQueryDao.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.entity; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -23,15 +22,10 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; -import java.util.List; -import java.util.Map; - public interface EntityQueryDao { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query); - Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index c4bf057efd..8f9226c180 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -53,7 +53,6 @@ import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -312,12 +311,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { EntityType entityType = resolveEntityType(query.getEntityFilter()); - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType)); + QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, entityType, TenantId.SYS_TENANT_ID.equals(tenantId))); if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); - ctx.append(" e where "); - ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList())); +// +// if (TenantId.SYS_TENANT_ID.equals(tenantId)) { +// ctx.append(" e"); +// } else { + ctx.append(" e where "); + ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList())); +// } + return transactionTemplate.execute(status -> { long startTs = System.currentTimeMillis(); try { @@ -389,61 +394,6 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { return findEntityDataByQuery(tenantId, customerId, query, false); } - @Override - public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { - int size = entityTypes.size(); - - QueryContext ctx = new QueryContext(new QuerySecurityContext(tenantId, customerId, null, true)); - ctx.append("select "); - - for (int i = 0; i < size; i++) { - ctx.append("(select count(*) from "); - ctx.append(getTableName(entityTypes.get(i))); - ctx.append(")"); - if (i < size - 1) { - ctx.append(", "); - } - } - - return transactionTemplate.execute(status -> { - long startTs = System.currentTimeMillis(); - try { - List counts = jdbcTemplate.query(ctx.getQuery(), rs -> { - List result = new ArrayList<>(); - if (rs.next()) { - for (int i = 1; i <= size; i++) { - result.add(rs.getLong(i)); - } - } - return result; - }); - - Map result = new HashMap<>(size); - for (int i = 0; i < size; i++) { - result.put(entityTypes.get(i), counts.get(i)); - } - return result; - } finally { - queryLog.logQuery(ctx, ctx.getQuery(), System.currentTimeMillis() - startTs); - } - }); - } - - private String getTableName(EntityType entityType) { - switch (entityType) { - case TENANT: - case TENANT_PROFILE: - case CUSTOMER: - case DEVICE: - case ASSET: - return entityType.name().toLowerCase(); - case USER: - return "tb_user"; - default: - throw new IllegalArgumentException("Not supported entity type: " + entityType + "!"); - } - } - public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, boolean ignorePermissionCheck) { return transactionTemplate.execute(status -> { EntityType entityType = resolveEntityType(query.getEntityFilter()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java index 5fd91057ce..7d78a34981 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityQueryRepository.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.sql.query; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -23,9 +22,6 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; -import java.util.List; -import java.util.Map; - public interface EntityQueryRepository { long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query); @@ -34,6 +30,4 @@ public interface EntityQueryRepository { PageData findEntityDataByQueryInternal(EntityDataQuery query); - Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java index 3929c98eec..192caed6a8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.query; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -26,9 +25,6 @@ import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.dao.entity.EntityQueryDao; -import java.util.List; -import java.util.Map; - @Component public class JpaEntityQueryDao implements EntityQueryDao { @@ -45,8 +41,4 @@ public class JpaEntityQueryDao implements EntityQueryDao { return entityQueryRepository.findEntityDataByQuery(tenantId, customerId, query); } - @Override - public Map countEntitiesByTypes(TenantId tenantId, CustomerId customerId, List entityTypes) { - return entityQueryRepository.countEntitiesByTypes(tenantId, customerId, entityTypes); - } } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 7eee546632..832f4c7046 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -1584,13 +1584,6 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { return restTemplate.postForObject(baseURL + "/api/entitiesQuery/count", query, Long.class); } - public Map countEntitiesByTypes(List entityTypes) { - return restTemplate.exchange( - baseURL + "/api/entitiesTypes/count", - HttpMethod.POST, new HttpEntity<>(entityTypes), new ParameterizedTypeReference>() { - }).getBody(); - } - public PageData findEntityDataByQuery(EntityDataQuery query) { return restTemplate.exchange( baseURL + "/api/entitiesQuery/find", From eea8f977b19f2b8acc941bdb426a7917d240c769 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 14 Mar 2023 16:44:31 +0100 Subject: [PATCH 06/16] added tenant profile type to the entityTableMap (EntityQuery) --- .../dao/sql/query/DefaultEntityQueryRepository.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 8f9226c180..2885776a6d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -241,6 +241,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.RULE_CHAIN, "rule_chain"); entityTableMap.put(EntityType.DEVICE_PROFILE, "device_profile"); entityTableMap.put(EntityType.ASSET_PROFILE, "asset_profile"); + entityTableMap.put(EntityType.TENANT_PROFILE, "tenant_profile"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ @@ -315,13 +316,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { if (query.getKeyFilters() == null || query.getKeyFilters().isEmpty()) { ctx.append("select count(e.id) from "); ctx.append(addEntityTableQuery(ctx, query.getEntityFilter())); -// -// if (TenantId.SYS_TENANT_ID.equals(tenantId)) { -// ctx.append(" e"); -// } else { - ctx.append(" e where "); - ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList())); -// } + ctx.append(" e where "); + ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList())); return transactionTemplate.execute(status -> { long startTs = System.currentTimeMillis(); From 8ede0b28226c97ad205886490138e0d966959386 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 15 Mar 2023 16:54:23 +0100 Subject: [PATCH 07/16] added ability to send system info throw ZK --- .../system/DefaultSystemInfoService.java | 116 ++++++++++++------ common/cluster-api/src/main/proto/queue.proto | 7 ++ .../server/common/data/SystemInfo.java | 12 +- .../server/common/data/SystemInfoData.java | 35 ++++++ .../DefaultTbServiceInfoProvider.java | 39 +++++- .../queue/discovery/DiscoveryService.java | 6 + .../discovery/DummyDiscoveryService.java | 6 + .../queue/discovery/HashPartitionService.java | 5 +- .../queue/discovery/PartitionService.java | 2 - .../discovery/TbServiceInfoProvider.java | 3 + .../queue/discovery/ZkDiscoveryService.java | 29 +++-- .../thingsboard/common/util/SystemUtil.java | 59 +++++++++ 12 files changed, 250 insertions(+), 69 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java create mode 100644 common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 3e9b778a5d..40ea65d9fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -15,77 +15,119 @@ */ package org.thingsboard.server.service.system; +import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ProtocolStringList; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.SystemInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.common.data.SystemInfoData; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; -import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; -import java.io.File; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryMXBean; -import java.lang.management.OperatingSystemMXBean; -import java.util.HashMap; +import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; -import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.thingsboard.common.util.SystemUtil.getCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; -@TbCoreComponent @Service @RequiredArgsConstructor +@Slf4j public class DefaultSystemInfoService implements SystemInfoService { + public static final FutureCallback CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Integer result) { + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to persist system info", t); + } + }; + private final TbServiceInfoProvider serviceInfoProvider; - private final PartitionService partitionService; + private final DiscoveryService discoveryService; + private final TelemetrySubscriptionService telemetryService; + private ScheduledExecutorService scheduler; @Value("${zk.enabled:false}") private boolean zkEnabled; + @PostConstruct + private void init() { + if (!zkEnabled) { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler")); + scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES); + } + } + + @PreDestroy + private void destroy() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + @Override - @SneakyThrows public SystemInfo getSystemInfo() { SystemInfo systemInfo = new SystemInfo(); - TransportProtos.ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfo(); - List currentOtherServices = partitionService.getCurrentOtherServices(); + ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfoWithCurrentSystemInfo(); if (zkEnabled) { - Map serviceInfos = new HashMap<>(); - addServiceInfo(serviceInfos, serviceInfo); - currentOtherServices.forEach(otherInfo -> addServiceInfo(serviceInfos, otherInfo)); - systemInfo.setServiceInfos(serviceInfos); + List clusterSystemData = new ArrayList<>(); + clusterSystemData.add(createSystemInfoData(serviceInfo)); + this.discoveryService.getOtherServers() + .stream() + .map(this::createSystemInfoData) + .forEach(clusterSystemData::add); + systemInfo.setSystemData(clusterSystemData); } else { systemInfo.setMonolith(true); - systemInfo.setMemUsage(getMemoryUsage()); - - systemInfo.setCpuUsage((int) (getCpuUsage() * 100) / 100.0); - systemInfo.setFreeDiscSpace(getFreeDiscSpace()); + systemInfo.setSystemData(Collections.singletonList(createSystemInfoData(serviceInfo))); } return systemInfo; } - private void addServiceInfo(Map serviceInfos, TransportProtos.ServiceInfo serviceInfo) { - ProtocolStringList serviceTypes = serviceInfo.getServiceTypesList(); - serviceInfos.put(serviceInfo.getServiceId(), serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0)); - } + private void saveCurrentSystemInfo() { + long ts = System.currentTimeMillis(); + List tsList = new ArrayList<>(); + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", getMemoryUsage()))); + tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", getCpuUsage()))); + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", getFreeDiscSpace()))); - private long getMemoryUsage() { - MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); - return memoryMXBean.getHeapMemoryUsage().getUsed(); + telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, tsList, CALLBACK); } - private double getCpuUsage() { - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - return osBean.getSystemLoadAverage(); + private SystemInfoData createSystemInfoData(ServiceInfo serviceInfo) { + ProtocolStringList serviceTypes = serviceInfo.getServiceTypesList(); + SystemInfoData infoData = new SystemInfoData(); + infoData.setServiceId(serviceInfo.getServiceId()); + infoData.setServiceType(serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0)); + infoData.setMemUsage(serviceInfo.getSystemInfo().getMemoryUsage()); + infoData.setCpuUsage(serviceInfo.getSystemInfo().getCpuUsage()); + infoData.setFreeDiscSpace(serviceInfo.getSystemInfo().getFreeDiscSpace()); + return infoData; } - private long getFreeDiscSpace() { - File file = new File("/"); - return file.getFreeSpace(); - } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index c1e6f4d1eb..24d1f67db3 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -27,6 +27,13 @@ message ServiceInfo { string serviceId = 1; repeated string serviceTypes = 2; repeated string transports = 6; + SystemInfoProto systemInfo = 10; +} + +message SystemInfoProto { + int64 memoryUsage = 1; + double cpuUsage = 2; + int64 freeDiscSpace = 3; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java index f1aa8b4abe..ea46c91239 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfo.java @@ -18,18 +18,12 @@ package org.thingsboard.server.common.data; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import java.util.Map; +import java.util.List; @Data public class SystemInfo { @ApiModelProperty(position = 1, value = "Is monolith.") private boolean isMonolith; - @ApiModelProperty(position = 2, value = "CPU usage.") - private Double cpuUsage; - @ApiModelProperty(position = 3, value = "Memory usage.") - private Long memUsage; - @ApiModelProperty(position = 4, value = "Free disc space.") - private Long freeDiscSpace; - @ApiModelProperty(position = 5, value = "Json object with info about services.") - private Map serviceInfos; + @ApiModelProperty(position = 2, value = "System data.") + private List systemData; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java new file mode 100644 index 0000000000..9345fb4feb --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + +@Data +public class SystemInfoData { + @ApiModelProperty(position = 1, value = "Service Id.") + private String serviceId; + @ApiModelProperty(position = 2, value = "Service type.") + private String serviceType; + @ApiModelProperty(position = 3, value = "CPU usage.") + private Double cpuUsage; + @ApiModelProperty(position = 4, value = "Memory usage.") + private Long memUsage; + @ApiModelProperty(position = 5, value = "Free disc space.") + private Long freeDiscSpace; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 0a9e00e409..b9377f3174 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.util.AfterContextReady; @@ -36,6 +37,10 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import static org.thingsboard.common.util.SystemUtil.getCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; + @Component @Slf4j public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @@ -69,11 +74,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } else { serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } - ServiceInfo.Builder builder = ServiceInfo.newBuilder() - .setServiceId(serviceId) - .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())); - serviceInfo = builder.build(); + serviceInfo = getServiceInfoWithCurrentSystemInfo(); } @AfterContextReady @@ -99,4 +101,33 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { return serviceTypes.contains(serviceType); } + @Override + public ServiceInfo getServiceInfoWithCurrentSystemInfo() { + ServiceInfo.Builder builder = ServiceInfo.newBuilder() + .setServiceId(serviceId) + .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())) + .setSystemInfo(getCurrentSystemInfoProto()); + + return builder.build(); + } + + private TransportProtos.SystemInfoProto getCurrentSystemInfoProto() { + TransportProtos.SystemInfoProto.Builder builder = TransportProtos.SystemInfoProto.newBuilder(); + + Long memoryUsage = getMemoryUsage(); + if (memoryUsage != null) { + builder.setMemoryUsage(memoryUsage); + } + Double cpuUsage = getCpuUsage(); + if (cpuUsage != null) { + builder.setCpuUsage(cpuUsage); + } + Long freeDiscSpace = getFreeDiscSpace(); + if (freeDiscSpace != null) { + builder.setFreeDiscSpace(freeDiscSpace); + } + + return builder.build(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java index 4bc65c5e1e..dfd6e7dba5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -15,6 +15,12 @@ */ package org.thingsboard.server.queue.discovery; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.List; + public interface DiscoveryService { + List getOtherServers(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index 7b04f3d0f6..b26284b1c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -22,9 +22,11 @@ import org.springframework.context.annotation.DependsOn; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.AfterStartUp; import java.util.Collections; +import java.util.List; @Service @ConditionalOnProperty(prefix = "zk", value = "enabled", havingValue = "false", matchIfMissing = true) @@ -46,4 +48,8 @@ public class DummyDiscoveryService implements DiscoveryService { partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), Collections.emptyList()); } + @Override + public List getOtherServers() { + return Collections.emptyList(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 6888b9ad8c..776d42a334 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -17,13 +17,11 @@ package org.thingsboard.server.queue.discovery; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -77,7 +75,6 @@ public class HashPartitionService implements PartitionService { private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); private Map> tbTransportServicesByType = new HashMap<>(); - @Getter private List currentOtherServices; private HashFunction hashFunction; @@ -219,7 +216,7 @@ public class HashPartitionService implements PartitionService { } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); - final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); + final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 3806c59a68..50e6fdf791 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -64,5 +63,4 @@ public interface PartitionService { void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); - List getCurrentOtherServices(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index 4b03056253..300bc05f0a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; public interface TbServiceInfoProvider { @@ -28,4 +29,6 @@ public interface TbServiceInfoProvider { boolean isService(ServiceType serviceType); + ServiceInfo getServiceInfoWithCurrentSystemInfo(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 63f91935d3..0a72f4064e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.discovery; import com.google.protobuf.InvalidProtocolBufferException; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -33,21 +34,19 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import org.thingsboard.server.queue.util.AfterStartUp; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.List; import java.util.NoSuchElementException; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent.Type.CHILD_REMOVED; @@ -71,7 +70,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private final TbServiceInfoProvider serviceInfoProvider; private final PartitionService partitionService; - private ExecutorService reconnectExecutorService; + private ScheduledExecutorService zkExecutorService; private CuratorFramework client; private PathChildrenCache cache; private String nodePath; @@ -93,7 +92,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi Assert.notNull(zkConnectionTimeout, missingProperty("zk.connection_timeout_ms")); Assert.notNull(zkSessionTimeout, missingProperty("zk.session_timeout_ms")); - reconnectExecutorService = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); + zkExecutorService = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("zk-discovery")); log.info("Initializing discovery service using ZK connect string: {}", zkUrl); @@ -101,7 +100,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi initZkClient(); } - private List getOtherServers() { + @Override + public List getOtherServers() { return cache.getCurrentData().stream() .filter(cd -> !cd.getPath().equals(nodePath)) .map(cd -> { @@ -128,15 +128,17 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return; } log.info("Going to publish current server..."); - publishCurrentServer(); + zkExecutorService.scheduleAtFixedRate(this::publishCurrentServer, 0, 1, TimeUnit.MINUTES); log.info("Going to recalculate partitions..."); recalculatePartitions(); } + @SneakyThrows public synchronized void publishCurrentServer() { TransportProtos.ServiceInfo self = serviceInfoProvider.getServiceInfo(); if (currentServerExists()) { - log.info("[{}] ZK node for current instance already exists, NOT created new one: {}", self.getServiceId(), nodePath); + log.trace("[{}] Updating ZK node for current instance: {}", self.getServiceId(), nodePath); + client.setData().forPath(nodePath, serviceInfoProvider.getServiceInfoWithCurrentSystemInfo().toByteArray()); } else { try { log.info("[{}] Creating ZK node for current instance", self.getServiceId()); @@ -175,7 +177,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return (client, newState) -> { log.info("[{}] ZK state changed: {}", self.getServiceId(), newState); if (newState == ConnectionState.LOST) { - reconnectExecutorService.submit(this::reconnect); + zkExecutorService.submit(this::reconnect); } }; } @@ -240,7 +242,7 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi @PreDestroy public void destroy() { destroyZkClient(); - reconnectExecutorService.shutdownNow(); + zkExecutorService.shutdownNow(); log.info("Stopped discovery service"); } @@ -280,13 +282,14 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi log.error("Failed to decode server instance for node {}", data.getPath(), e); throw e; } - log.info("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); + log.debug("Processing [{}] event for [{}]", pathChildrenCacheEvent.getType(), instance.getServiceId()); switch (pathChildrenCacheEvent.getType()) { case CHILD_ADDED: - case CHILD_UPDATED: case CHILD_REMOVED: recalculatePartitions(); break; + case CHILD_UPDATED: + break; default: break; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java new file mode 100644 index 0000000000..4533bbf65e --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import lombok.extern.slf4j.Slf4j; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.OperatingSystemMXBean; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Paths; + +@Slf4j +public class SystemUtil { + + public static Long getMemoryUsage() { + try { + MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); + return memoryMXBean.getHeapMemoryUsage().getUsed(); + } catch (Exception e) { + log.debug("Failed to get memory usage!!!", e); + } + return null; + } + + public static Double getCpuUsage() { + try { + OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); + return osBean.getSystemLoadAverage(); + } catch (Exception e) { + log.debug("Failed to get cpu usage!!!", e); + } + return null; + } + + public static Long getFreeDiscSpace() { + try { + FileStore store = Files.getFileStore(Paths.get("/")); + return store.getUsableSpace(); + } catch (Exception e) { + log.debug("Failed to get free disc space!!!", e); + } + return null; + } +} From 4cb2a304eeba512e94dbaf9797b65ba591f9a3a6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 15 Mar 2023 17:06:05 +0100 Subject: [PATCH 08/16] added getSystemInfo to the rest client --- .../main/java/org/thingsboard/rest/client/RestClient.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 832f4c7046..33524a9fa1 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; @@ -373,6 +374,10 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } } + public SystemInfo getSystemInfo() { + return restTemplate.getForEntity(baseURL + "/api/admin/systemInfo", SystemInfo.class).getBody(); + } + public Optional getAlarmById(AlarmId alarmId) { try { ResponseEntity alarm = restTemplate.getForEntity(baseURL + "/api/alarm/{alarmId}", Alarm.class, alarmId.getId()); From 118fc1554a0ee4732bb26e649655f574b6200088 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 15 Mar 2023 18:01:29 +0100 Subject: [PATCH 09/16] added TbCoreComponent to the SystemInfoService --- .../server/service/query/DefaultEntityQueryService.java | 2 +- .../server/service/system/DefaultSystemInfoService.java | 2 ++ .../java/org/thingsboard/server/dao/entity/EntityService.java | 2 +- .../thingsboard/server/queue/discovery/PartitionService.java | 1 - .../org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java | 1 - 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index ba5b29c11b..5a9d0d9edf 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -29,7 +29,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; @@ -56,6 +55,7 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.security.AccessValidator; diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 40ea65d9fd..0c0f74a199 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; @@ -48,6 +49,7 @@ import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +@TbCoreComponent @Service @RequiredArgsConstructor @Slf4j diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java index 3983494d70..f2317efa05 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityService.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.dao.entity; +import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.NameLabelAndCustomerDetails; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index 50e6fdf791..2ff65ef07b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -62,5 +62,4 @@ public interface PartitionService { void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg); void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java index 192caed6a8..33d9426bdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java @@ -40,5 +40,4 @@ public class JpaEntityQueryDao implements EntityQueryDao { public PageData findEntityDataByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { return entityQueryRepository.findEntityDataByQuery(tenantId, customerId, query); } - } From d13a458eb07c4651e682af9549c8298ae3d762e3 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 16 Mar 2023 10:21:40 +0100 Subject: [PATCH 10/16] saveCUrrentSystemInfo improvements --- .../system/DefaultSystemInfoService.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 0c0f74a199..41d2f6cdbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -111,12 +111,22 @@ public class DefaultSystemInfoService implements SystemInfoService { return systemInfo; } - private void saveCurrentSystemInfo() { + protected void saveCurrentSystemInfo() { long ts = System.currentTimeMillis(); List tsList = new ArrayList<>(); - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", getMemoryUsage()))); - tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", getCpuUsage()))); - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", getFreeDiscSpace()))); + + Long memoryUsage = getMemoryUsage(); + if (memoryUsage != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", memoryUsage))); + } + Double cpuUsage = getCpuUsage(); + if (cpuUsage != null) { + tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", cpuUsage))); + } + Long freeDiscSpace = getFreeDiscSpace(); + if (freeDiscSpace != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", freeDiscSpace))); + } telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, tsList, CALLBACK); } From 671c7db5c9347f99b0010ad26b1117c708531d3c Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 21 Mar 2023 17:19:51 +0100 Subject: [PATCH 11/16] added new metrics --- .../system/DefaultSystemInfoService.java | 104 ++++++++++++++---- common/cluster-api/src/main/proto/queue.proto | 10 +- .../server/common/data/SystemInfoData.java | 14 ++- .../DefaultTbServiceInfoProvider.java | 20 ++++ common/util/pom.xml | 4 + .../thingsboard/common/util/SystemUtil.java | 54 ++++++++- pom.xml | 6 + 7 files changed, 180 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 41d2f6cdbf..8f05f26648 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -21,22 +21,27 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.SystemInfoData; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.Collections; @@ -45,15 +50,19 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @TbCoreComponent @Service @RequiredArgsConstructor @Slf4j -public class DefaultSystemInfoService implements SystemInfoService { +public class DefaultSystemInfoService extends TbApplicationEventListener implements SystemInfoService { public static final FutureCallback CALLBACK = new FutureCallback<>() { @Override @@ -67,6 +76,7 @@ public class DefaultSystemInfoService implements SystemInfoService { }; private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; private final DiscoveryService discoveryService; private final TelemetrySubscriptionService telemetryService; private ScheduledExecutorService scheduler; @@ -74,18 +84,15 @@ public class DefaultSystemInfoService implements SystemInfoService { @Value("${zk.enabled:false}") private boolean zkEnabled; - @PostConstruct - private void init() { - if (!zkEnabled) { - scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler")); - scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES); - } - } - - @PreDestroy - private void destroy() { - if (scheduler != null) { - scheduler.shutdownNow(); + @Override + protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) { + if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) { + if (scheduler == null && partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition()) { + scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler")); + scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES); + } else { + destroy(); + } } } @@ -96,13 +103,7 @@ public class DefaultSystemInfoService implements SystemInfoService { ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfoWithCurrentSystemInfo(); if (zkEnabled) { - List clusterSystemData = new ArrayList<>(); - clusterSystemData.add(createSystemInfoData(serviceInfo)); - this.discoveryService.getOtherServers() - .stream() - .map(this::createSystemInfoData) - .forEach(clusterSystemData::add); - systemInfo.setSystemData(clusterSystemData); + systemInfo.setSystemData(getSystemData(serviceInfo)); } else { systemInfo.setMonolith(true); systemInfo.setSystemData(Collections.singletonList(createSystemInfoData(serviceInfo))); @@ -112,6 +113,22 @@ public class DefaultSystemInfoService implements SystemInfoService { } protected void saveCurrentSystemInfo() { + if (zkEnabled) { + saveCurrentClusterSystemInfo(); + } else { + saveCurrentMonolithSystemInfo(); + } + } + + private void saveCurrentClusterSystemInfo() { + long ts = System.currentTimeMillis(); + + List clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfoWithCurrentSystemInfo()); + BasicTsKvEntry clusterDataKv = new BasicTsKvEntry(ts, new JsonDataEntry("clusterSystemData", JacksonUtil.toString(clusterSystemData))); + doSave(Collections.singletonList(clusterDataKv)); + } + + private void saveCurrentMonolithSystemInfo() { long ts = System.currentTimeMillis(); List tsList = new ArrayList<>(); @@ -119,16 +136,46 @@ public class DefaultSystemInfoService implements SystemInfoService { if (memoryUsage != null) { tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", memoryUsage))); } + Long totalMemory = getTotalMemory(); + if (totalMemory != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalMemory", totalMemory))); + } + Long freeMemory = getFreeMemory(); + if (freeMemory != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeMemory", freeMemory))); + } Double cpuUsage = getCpuUsage(); if (cpuUsage != null) { tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", cpuUsage))); } + Double totalCpuUsage = getTotalCpuUsage(); + if (totalCpuUsage != null) { + tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("totalCpuUsage", totalCpuUsage))); + } Long freeDiscSpace = getFreeDiscSpace(); if (freeDiscSpace != null) { tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", freeDiscSpace))); } + Long totalDiscSpace = getTotalDiscSpace(); + if (totalDiscSpace != null) { + tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalDiscSpace", totalDiscSpace))); + } + + doSave(tsList); + } - telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, tsList, CALLBACK); + private void doSave(List telemetry) { + telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, telemetry, CALLBACK); + } + + private List getSystemData(ServiceInfo serviceInfo) { + List clusterSystemData = new ArrayList<>(); + clusterSystemData.add(createSystemInfoData(serviceInfo)); + this.discoveryService.getOtherServers() + .stream() + .map(this::createSystemInfoData) + .forEach(clusterSystemData::add); + return clusterSystemData; } private SystemInfoData createSystemInfoData(ServiceInfo serviceInfo) { @@ -136,10 +183,21 @@ public class DefaultSystemInfoService implements SystemInfoService { SystemInfoData infoData = new SystemInfoData(); infoData.setServiceId(serviceInfo.getServiceId()); infoData.setServiceType(serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0)); - infoData.setMemUsage(serviceInfo.getSystemInfo().getMemoryUsage()); + infoData.setMemoryUsage(serviceInfo.getSystemInfo().getMemoryUsage()); + infoData.setTotalMemory(serviceInfo.getSystemInfo().getTotalMemory()); + infoData.setFreeMemory(serviceInfo.getSystemInfo().getFreeMemory()); infoData.setCpuUsage(serviceInfo.getSystemInfo().getCpuUsage()); + infoData.setTotalCpuUsage(serviceInfo.getSystemInfo().getTotalCpuUsage()); infoData.setFreeDiscSpace(serviceInfo.getSystemInfo().getFreeDiscSpace()); + infoData.setTotalDiscSpace(serviceInfo.getSystemInfo().getTotalDiscSpace()); return infoData; } + @PreDestroy + private void destroy() { + if (scheduler != null) { + scheduler.shutdownNow(); + scheduler = null; + } + } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 24d1f67db3..0af667356d 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -31,9 +31,13 @@ message ServiceInfo { } message SystemInfoProto { - int64 memoryUsage = 1; - double cpuUsage = 2; - int64 freeDiscSpace = 3; + double cpuUsage = 1; + double totalCpuUsage = 2; + int64 memoryUsage = 3; + int64 totalMemory = 4; + int64 freeMemory = 5; + int64 freeDiscSpace = 6; + int64 totalDiscSpace = 7; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java index 9345fb4feb..e4510e0f48 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java @@ -28,8 +28,16 @@ public class SystemInfoData { private String serviceType; @ApiModelProperty(position = 3, value = "CPU usage.") private Double cpuUsage; - @ApiModelProperty(position = 4, value = "Memory usage.") - private Long memUsage; - @ApiModelProperty(position = 5, value = "Free disc space.") + @ApiModelProperty(position = 4, value = "Total CPU usage.") + private Double totalCpuUsage; + @ApiModelProperty(position = 5, value = "Memory usage in bytes.") + private Long memoryUsage; + @ApiModelProperty(position = 6, value = "Total memory in bytes.") + private Long totalMemory; + @ApiModelProperty(position = 6, value = "Free memory in bytes.") + private Long freeMemory; + @ApiModelProperty(position = 7, value = "Free disc space in bytes.") private Long freeDiscSpace; + @ApiModelProperty(position = 7, value = "Total disc space in bytes.") + private Long totalDiscSpace; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index b9377f3174..db9334d3a0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -37,9 +37,13 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; +import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getTotalMemory; @Component @Slf4j @@ -118,14 +122,30 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { if (memoryUsage != null) { builder.setMemoryUsage(memoryUsage); } + Long totalMemory = getTotalMemory(); + if (totalMemory != null) { + builder.setTotalMemory(totalMemory); + } + Long freeMemory = getFreeMemory(); + if (freeMemory != null) { + builder.setFreeMemory(freeMemory); + } Double cpuUsage = getCpuUsage(); if (cpuUsage != null) { builder.setCpuUsage(cpuUsage); } + Double totalCpuUsage = getTotalCpuUsage(); + if (totalCpuUsage != null) { + builder.setTotalCpuUsage(totalCpuUsage); + } Long freeDiscSpace = getFreeDiscSpace(); if (freeDiscSpace != null) { builder.setFreeDiscSpace(freeDiscSpace); } + Long totalDiscSpace = getTotalDiscSpace(); + if (totalDiscSpace != null) { + builder.setTotalDiscSpace(totalDiscSpace); + } return builder.build(); } diff --git a/common/util/pom.xml b/common/util/pom.xml index a222724acd..18b08621ec 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -88,6 +88,10 @@ org.thingsboard.common data + + com.github.dblock + oshi-core + diff --git a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java index 4533bbf65e..18ddf1a0a0 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java @@ -16,10 +16,11 @@ package org.thingsboard.common.util; import lombok.extern.slf4j.Slf4j; +import oshi.SystemInfo; +import oshi.hardware.HardwareAbstractionLayer; import java.lang.management.ManagementFactory; import java.lang.management.MemoryMXBean; -import java.lang.management.OperatingSystemMXBean; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; @@ -27,6 +28,13 @@ import java.nio.file.Paths; @Slf4j public class SystemUtil { + private static final HardwareAbstractionLayer HARDWARE; + + static { + SystemInfo si = new SystemInfo(); + HARDWARE = si.getHardware(); + } + public static Long getMemoryUsage() { try { MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); @@ -37,16 +45,42 @@ public class SystemUtil { return null; } + public static Long getTotalMemory() { + try { + return HARDWARE.getMemory().getTotal(); + } catch (Exception e) { + log.debug("Failed to get total memory!!!", e); + } + return null; + } + + public static Long getFreeMemory() { + try { + return HARDWARE.getMemory().getAvailable(); + } catch (Exception e) { + log.debug("Failed to get free memory!!!", e); + } + return null; + } + public static Double getCpuUsage() { try { - OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); - return osBean.getSystemLoadAverage(); + return prepare(HARDWARE.getProcessor().getSystemLoadAverage()); } catch (Exception e) { log.debug("Failed to get cpu usage!!!", e); } return null; } + public static Double getTotalCpuUsage() { + try { + return prepare(HARDWARE.getProcessor().getSystemCpuLoad() * 100); + } catch (Exception e) { + log.debug("Failed to get total cpu usage!!!", e); + } + return null; + } + public static Long getFreeDiscSpace() { try { FileStore store = Files.getFileStore(Paths.get("/")); @@ -56,4 +90,18 @@ public class SystemUtil { } return null; } + + public static Long getTotalDiscSpace() { + try { + FileStore store = Files.getFileStore(Paths.get("/")); + return store.getTotalSpace(); + } catch (Exception e) { + log.debug("Failed to get total disc space!!!", e); + } + return null; + } + + private static Double prepare(Double d) { + return (int) (d * 100) / 100.0; + } } diff --git a/pom.xml b/pom.xml index 65d9de814e..7b6d565ead 100755 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,7 @@ 2.21.0 2.12.0 1.12.1 + 3.4.0 @@ -1987,6 +1988,11 @@ exp4j ${exp4j.version} + + com.github.dblock + oshi-core + ${oshi.version} + From befa454017aba672ff1e18d06419db0e73928d2b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 22 Mar 2023 13:18:43 +0100 Subject: [PATCH 12/16] save system info to api usage, and added tests --- .../system/DefaultSystemInfoService.java | 8 +- .../controller/BaseHomePageApiTest.java | 241 ++++++++++++++++++ .../controller/sql/HomePageApiSqlTest.java | 23 ++ 3 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/sql/HomePageApiSqlTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 8f05f26648..6882157000 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.SystemInfoData; import org.thingsboard.server.common.data.id.TenantId; @@ -32,6 +33,7 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.stats.TbApiUsageStateClient; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; @@ -50,9 +52,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; @@ -79,6 +81,7 @@ public class DefaultSystemInfoService extends TbApplicationEventListener telemetry) { - telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, telemetry, CALLBACK); + ApiUsageState apiUsageState = apiUsageStateClient.getApiUsageState(TenantId.SYS_TENANT_ID); + telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, apiUsageState.getId(), telemetry, CALLBACK); } private List getSystemData(ServiceInfo serviceInfo) { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java new file mode 100644 index 0000000000..07fa2ddaad --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java @@ -0,0 +1,241 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.stats.TbApiUsageStateClient; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; +import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class BaseHomePageApiTest extends AbstractControllerTest { + + @Autowired + private TbApiUsageStateClient apiUsageStateClient; + + //For system administrator + @Test + public void testTenantsCountWsCmd() throws Exception { + loginSysAdmin(); + + List tenants = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Tenant tenant = new Tenant(); + tenant.setTitle("tenant" + i); + tenants.add(doPost("/api/tenant", tenant, Tenant.class)); + } + + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.TENANT); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(101, update.getCount()); + + for (Tenant tenant : tenants) { + doDelete("/api/tenant/" + tenant.getId().toString()); + } + } + + @Test + public void testTenantProfilesCountWsCmd() throws Exception { + loginSysAdmin(); + + List tenantProfiles = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("tenantProfile" + i); + tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); + } + + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.TENANT_PROFILE); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(101, update.getCount()); + + for (TenantProfile tenantProfile : tenantProfiles) { + doDelete("/api/tenantProfile/" + tenantProfile.getId().toString()); + } + } + + @Test + public void testUsersCountWsCmd() throws Exception { + loginSysAdmin(); + + List users = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + User user = new User(); + user.setEmail(i + "user@thingsboard.org"); + user.setTenantId(tenantId); + user.setAuthority(Authority.TENANT_ADMIN); + users.add(doPost("/api/user", user, User.class)); + } + + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.USER); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(103, update.getCount()); + + for (User user : users) { + doDelete("/api/user/" + user.getId().toString()); + } + } + + @Test + public void testCustomersCountWsCmd() throws Exception { + loginTenantAdmin(); + + List customers = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Customer customer = new Customer(); + customer.setTitle("customer" + i); + customers.add(doPost("/api/customer", customer, Customer.class)); + } + + loginSysAdmin(); + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.CUSTOMER); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(101, update.getCount()); + + loginTenantAdmin(); + for (Customer customer : customers) { + doDelete("/api/customer/" + customer.getId().toString()); + } + } + + @Test + public void testDevicesCountWsCmd() throws Exception { + loginTenantAdmin(); + + List devices = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Device device = new Device(); + device.setName("device" + i); + devices.add(doPost("/api/device", device, Device.class)); + } + + loginSysAdmin(); + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.DEVICE); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(100, update.getCount()); + + loginTenantAdmin(); + for (Device device : devices) { + doDelete("/api/device/" + device.getId().toString()); + } + } + + @Test + public void testAssetsCountWsCmd() throws Exception { + loginTenantAdmin(); + + List assets = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + Asset asset = new Asset(); + asset.setName("asset" + i); + assets.add(doPost("/api/asset", asset, Asset.class)); + } + + loginSysAdmin(); + EntityTypeFilter ef = new EntityTypeFilter(); + ef.setEntityType(EntityType.ASSET); + EntityCountCmd cmd = new EntityCountCmd(1, new EntityCountQuery(ef, Collections.emptyList())); + getWsClient().send(cmd); + EntityCountUpdate update = getWsClient().parseCountReply(getWsClient().waitForReply()); + Assert.assertEquals(1, update.getCmdId()); + Assert.assertEquals(100, update.getCount()); + + loginTenantAdmin(); + for (Asset asset : assets) { + doDelete("/api/asset/" + asset.getId().toString()); + } + } + + @Test + public void testSystemInfoTimeSeriesWsCmd() throws Exception { + ApiUsageState apiUsageState = apiUsageStateClient.getApiUsageState(TenantId.SYS_TENANT_ID); + Assert.assertNotNull(apiUsageState); + + SingleEntityFilter entityFilter = new SingleEntityFilter(); + entityFilter.setSingleEntity(apiUsageState.getId()); + + loginSysAdmin(); + long now = System.currentTimeMillis(); + + EntityDataUpdate update = getWsClient().sendEntityDataQuery(entityFilter); + + Assert.assertEquals(1, update.getCmdId()); + PageData pageData = update.getData(); + Assert.assertNotNull(pageData); + Assert.assertEquals(1, pageData.getData().size()); + Assert.assertEquals(apiUsageState.getId(), pageData.getData().get(0).getEntityId()); + + update = getWsClient().subscribeTsUpdate( + List.of("memoryUsage", "totalMemory", "freeMemory", "cpuUsage", "totalCpuUsage", "freeDiscSpace", "totalDiscSpace"), + now, TimeUnit.HOURS.toMillis(1)); + Assert.assertEquals(1, update.getCmdId()); + List listData = update.getUpdate(); + Assert.assertNotNull(listData); + Assert.assertEquals(1, listData.size()); + Assert.assertEquals(apiUsageState.getId(), listData.get(0).getEntityId()); + Assert.assertEquals(7, listData.get(0).getTimeseries().size()); + + for (TsValue[] tsv : listData.get(0).getTimeseries().values()) { + Assert.assertEquals(1, tsv.length); + } + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/HomePageApiSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/HomePageApiSqlTest.java new file mode 100644 index 0000000000..5aa50c9336 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/sql/HomePageApiSqlTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller.sql; + +import org.thingsboard.server.controller.BaseHomePageApiTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@DaoSqlTest +public class HomePageApiSqlTest extends BaseHomePageApiTest { +} From 32278beabc762293265820d2af159826e875ee22 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 23 Mar 2023 13:56:36 +0100 Subject: [PATCH 13/16] improvements due to comments --- .../system/DefaultSystemInfoService.java | 68 +++++++------------ .../DefaultTbServiceInfoProvider.java | 37 +++------- .../queue/discovery/DiscoveryService.java | 2 + .../discovery/DummyDiscoveryService.java | 5 ++ .../queue/discovery/ZkDiscoveryService.java | 7 +- .../thingsboard/common/util/SystemUtil.java | 43 ++++++------ 6 files changed, 65 insertions(+), 97 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 6882157000..9b251687f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ProtocolStringList; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -82,19 +81,20 @@ public class DefaultSystemInfoService extends TbApplicationEventListener clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfoWithCurrentSystemInfo()); BasicTsKvEntry clusterDataKv = new BasicTsKvEntry(ts, new JsonDataEntry("clusterSystemData", JacksonUtil.toString(clusterSystemData))); doSave(Collections.singletonList(clusterDataKv)); @@ -135,34 +134,13 @@ public class DefaultSystemInfoService extends TbApplicationEventListener tsList = new ArrayList<>(); - Long memoryUsage = getMemoryUsage(); - if (memoryUsage != null) { - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", memoryUsage))); - } - Long totalMemory = getTotalMemory(); - if (totalMemory != null) { - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalMemory", totalMemory))); - } - Long freeMemory = getFreeMemory(); - if (freeMemory != null) { - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeMemory", freeMemory))); - } - Double cpuUsage = getCpuUsage(); - if (cpuUsage != null) { - tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", cpuUsage))); - } - Double totalCpuUsage = getTotalCpuUsage(); - if (totalCpuUsage != null) { - tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("totalCpuUsage", totalCpuUsage))); - } - Long freeDiscSpace = getFreeDiscSpace(); - if (freeDiscSpace != null) { - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", freeDiscSpace))); - } - Long totalDiscSpace = getTotalDiscSpace(); - if (totalDiscSpace != null) { - tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalDiscSpace", totalDiscSpace))); - } + getMemoryUsage().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", v)))); + getTotalMemory().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalMemory", v)))); + getFreeMemory().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeMemory", v)))); + getCpuUsage().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", v)))); + getTotalCpuUsage().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("totalCpuUsage", v)))); + getFreeDiscSpace().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", v)))); + getTotalDiscSpace().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalDiscSpace", v)))); doSave(tsList); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index db9334d3a0..a9eeb8ace0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -37,9 +37,9 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getCpuUsage; import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace; +import static org.thingsboard.common.util.SystemUtil.getFreeMemory; import static org.thingsboard.common.util.SystemUtil.getMemoryUsage; import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage; import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace; @@ -118,34 +118,13 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { private TransportProtos.SystemInfoProto getCurrentSystemInfoProto() { TransportProtos.SystemInfoProto.Builder builder = TransportProtos.SystemInfoProto.newBuilder(); - Long memoryUsage = getMemoryUsage(); - if (memoryUsage != null) { - builder.setMemoryUsage(memoryUsage); - } - Long totalMemory = getTotalMemory(); - if (totalMemory != null) { - builder.setTotalMemory(totalMemory); - } - Long freeMemory = getFreeMemory(); - if (freeMemory != null) { - builder.setFreeMemory(freeMemory); - } - Double cpuUsage = getCpuUsage(); - if (cpuUsage != null) { - builder.setCpuUsage(cpuUsage); - } - Double totalCpuUsage = getTotalCpuUsage(); - if (totalCpuUsage != null) { - builder.setTotalCpuUsage(totalCpuUsage); - } - Long freeDiscSpace = getFreeDiscSpace(); - if (freeDiscSpace != null) { - builder.setFreeDiscSpace(freeDiscSpace); - } - Long totalDiscSpace = getTotalDiscSpace(); - if (totalDiscSpace != null) { - builder.setTotalDiscSpace(totalDiscSpace); - } + getMemoryUsage().ifPresent(builder::setMemoryUsage); + getTotalMemory().ifPresent(builder::setTotalMemory); + getFreeMemory().ifPresent(builder::setFreeMemory); + getCpuUsage().ifPresent(builder::setCpuUsage); + getTotalCpuUsage().ifPresent(builder::setTotalCpuUsage); + getFreeDiscSpace().ifPresent(builder::setFreeDiscSpace); + getTotalDiscSpace().ifPresent(builder::setTotalDiscSpace); return builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java index dfd6e7dba5..e6aa604d00 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DiscoveryService.java @@ -23,4 +23,6 @@ public interface DiscoveryService { List getOtherServers(); + boolean isMonolith(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java index b26284b1c4..8e2599ae34 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DummyDiscoveryService.java @@ -52,4 +52,9 @@ public class DummyDiscoveryService implements DiscoveryService { public List getOtherServers() { return Collections.emptyList(); } + + @Override + public boolean isMonolith() { + return true; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 0a72f4064e..a14c2e4a4f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -115,6 +115,11 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi .collect(Collectors.toList()); } + @Override + public boolean isMonolith() { + return false; + } + @AfterStartUp(order = AfterStartUp.DISCOVERY_SERVICE) public void onApplicationEvent(ApplicationReadyEvent event) { if (stopped) { @@ -288,8 +293,6 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi case CHILD_REMOVED: recalculatePartitions(); break; - case CHILD_UPDATED: - break; default: break; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java index 18ddf1a0a0..686cfb28f2 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java @@ -24,6 +24,7 @@ import java.lang.management.MemoryMXBean; import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Optional; @Slf4j public class SystemUtil { @@ -35,70 +36,70 @@ public class SystemUtil { HARDWARE = si.getHardware(); } - public static Long getMemoryUsage() { + public static Optional getMemoryUsage() { try { MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean(); - return memoryMXBean.getHeapMemoryUsage().getUsed(); + return Optional.of(memoryMXBean.getHeapMemoryUsage().getUsed()); } catch (Exception e) { log.debug("Failed to get memory usage!!!", e); } - return null; + return Optional.empty(); } - public static Long getTotalMemory() { + public static Optional getTotalMemory() { try { - return HARDWARE.getMemory().getTotal(); + return Optional.of(HARDWARE.getMemory().getTotal()); } catch (Exception e) { log.debug("Failed to get total memory!!!", e); } - return null; + return Optional.empty(); } - public static Long getFreeMemory() { + public static Optional getFreeMemory() { try { - return HARDWARE.getMemory().getAvailable(); + return Optional.of(HARDWARE.getMemory().getAvailable()); } catch (Exception e) { log.debug("Failed to get free memory!!!", e); } - return null; + return Optional.empty(); } - public static Double getCpuUsage() { + public static Optional getCpuUsage() { try { - return prepare(HARDWARE.getProcessor().getSystemLoadAverage()); + return Optional.of(prepare(HARDWARE.getProcessor().getSystemLoadAverage())); } catch (Exception e) { log.debug("Failed to get cpu usage!!!", e); } - return null; + return Optional.empty(); } - public static Double getTotalCpuUsage() { + public static Optional getTotalCpuUsage() { try { - return prepare(HARDWARE.getProcessor().getSystemCpuLoad() * 100); + return Optional.of(prepare(HARDWARE.getProcessor().getSystemCpuLoad() * 100)); } catch (Exception e) { log.debug("Failed to get total cpu usage!!!", e); } - return null; + return Optional.empty(); } - public static Long getFreeDiscSpace() { + public static Optional getFreeDiscSpace() { try { FileStore store = Files.getFileStore(Paths.get("/")); - return store.getUsableSpace(); + return Optional.of(store.getUsableSpace()); } catch (Exception e) { log.debug("Failed to get free disc space!!!", e); } - return null; + return Optional.empty(); } - public static Long getTotalDiscSpace() { + public static Optional getTotalDiscSpace() { try { FileStore store = Files.getFileStore(Paths.get("/")); - return store.getTotalSpace(); + return Optional.of(store.getTotalSpace()); } catch (Exception e) { log.debug("Failed to get total disc space!!!", e); } - return null; + return Optional.empty(); } private static Double prepare(Double d) { From 37a5aeb485713b67370a6cd78dfc9c33301361fd Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 23 Mar 2023 16:34:51 +0100 Subject: [PATCH 14/16] implemented features info --- .../server/controller/AdminController.java | 13 ++ .../system/DefaultSystemInfoService.java | 30 ++++ .../service/system/SystemInfoService.java | 3 + .../controller/BaseHomePageApiTest.java | 134 ++++++++++++++++++ .../server/common/data/FeaturesInfo.java | 27 ++++ .../server/dao/service/DataValidator.java | 2 +- 6 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/FeaturesInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index a8eee51f17..8436463afb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -39,6 +39,9 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.FeaturesInfo; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -403,4 +406,14 @@ public class AdminController extends BaseController { return systemInfoService.getSystemInfo(); } + @ApiOperation(value = "Get features info (getFeaturesInfo)", + notes = "Get information about enabled/disabled features. " + + SYSTEM_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/featuresInfo", method = RequestMethod.GET) + @ResponseBody + public FeaturesInfo getFeaturesInfo() { + return systemInfoService.getFeaturesInfo(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index 9b251687f4..e6f98af4a1 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.system; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.util.concurrent.FutureCallback; import com.google.protobuf.ProtocolStringList; import lombok.RequiredArgsConstructor; @@ -22,7 +23,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.FeaturesInfo; import org.thingsboard.server.common.data.SystemInfo; import org.thingsboard.server.common.data.SystemInfoData; import org.thingsboard.server.common.data.id.TenantId; @@ -33,6 +36,9 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.stats.TbApiUsageStateClient; +import org.thingsboard.server.dao.oauth2.OAuth2Service; +import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; @@ -81,6 +87,8 @@ public class DefaultSystemInfoService extends TbApplicationEventListener clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfoWithCurrentSystemInfo()); diff --git a/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java index 3b48d809b2..1ca0fb8fe0 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/SystemInfoService.java @@ -15,8 +15,11 @@ */ package org.thingsboard.server.service.system; +import org.thingsboard.server.common.data.FeaturesInfo; import org.thingsboard.server.common.data.SystemInfo; public interface SystemInfoService { SystemInfo getSystemInfo(); + + FeaturesInfo getFeaturesInfo(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java index 07fa2ddaad..c2bf7b253c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java @@ -15,19 +15,33 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.FeaturesInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.MapperType; +import org.thingsboard.server.common.data.oauth2.OAuth2CustomMapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2DomainInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2Info; +import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; +import org.thingsboard.server.common.data.oauth2.OAuth2ParamsInfo; +import org.thingsboard.server.common.data.oauth2.OAuth2RegistrationInfo; +import org.thingsboard.server.common.data.oauth2.SchemeType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; @@ -41,10 +55,14 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + @Slf4j public abstract class BaseHomePageApiTest extends AbstractControllerTest { @@ -238,4 +256,120 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { } } + @Test + public void testGetFeaturesInfo() throws Exception { + loginSysAdmin(); + + FeaturesInfo featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertNotNull(featuresInfo); + Assert.assertFalse(featuresInfo.isEmailEnabled()); + Assert.assertFalse(featuresInfo.isSmsEnabled()); + Assert.assertFalse(featuresInfo.isTwoFaEnabled()); + Assert.assertFalse(featuresInfo.isNotificationEnabled()); + Assert.assertFalse(featuresInfo.isOauthEnabled()); + + AdminSettings mailSettings = doGet("/api/admin/settings/mail", AdminSettings.class); + + JsonNode jsonValue = mailSettings.getJsonValue(); + ((ObjectNode) jsonValue).put("mailFrom", "test@thingsboard.org"); + mailSettings.setJsonValue(jsonValue); + + doPost("/api/admin/settings", mailSettings).andExpect(status().isOk()); + + featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertTrue(featuresInfo.isEmailEnabled()); + Assert.assertFalse(featuresInfo.isSmsEnabled()); + Assert.assertFalse(featuresInfo.isTwoFaEnabled()); + Assert.assertFalse(featuresInfo.isNotificationEnabled()); + Assert.assertFalse(featuresInfo.isOauthEnabled()); + + AdminSettings smsSettings = new AdminSettings(); + smsSettings.setKey("sms"); + smsSettings.setJsonValue(JacksonUtil.newObjectNode()); + doPost("/api/admin/settings", smsSettings).andExpect(status().isOk()); + + featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertTrue(featuresInfo.isEmailEnabled()); + Assert.assertTrue(featuresInfo.isSmsEnabled()); + Assert.assertFalse(featuresInfo.isTwoFaEnabled()); + Assert.assertFalse(featuresInfo.isNotificationEnabled()); + Assert.assertFalse(featuresInfo.isOauthEnabled()); + + AdminSettings twoFaSettingsSettings = new AdminSettings(); + twoFaSettingsSettings.setKey("twoFaSettings"); + twoFaSettingsSettings.setJsonValue(JacksonUtil.newObjectNode()); + doPost("/api/admin/settings", twoFaSettingsSettings).andExpect(status().isOk()); + + featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertTrue(featuresInfo.isEmailEnabled()); + Assert.assertTrue(featuresInfo.isSmsEnabled()); + Assert.assertTrue(featuresInfo.isTwoFaEnabled()); + Assert.assertFalse(featuresInfo.isNotificationEnabled()); + Assert.assertFalse(featuresInfo.isOauthEnabled()); + + AdminSettings notificationsSettings = new AdminSettings(); + notificationsSettings.setKey("notifications"); + notificationsSettings.setJsonValue(JacksonUtil.newObjectNode()); + doPost("/api/admin/settings", notificationsSettings).andExpect(status().isOk()); + + featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertTrue(featuresInfo.isEmailEnabled()); + Assert.assertTrue(featuresInfo.isSmsEnabled()); + Assert.assertTrue(featuresInfo.isTwoFaEnabled()); + Assert.assertTrue(featuresInfo.isNotificationEnabled()); + Assert.assertFalse(featuresInfo.isOauthEnabled()); + + OAuth2Info oAuth2Info = createDefaultOAuth2Info(); + + doPost("/api/oauth2/config", oAuth2Info).andExpect(status().isOk()); + + featuresInfo = doGet("/api/admin/featuresInfo", FeaturesInfo.class); + Assert.assertNotNull(featuresInfo); + Assert.assertTrue(featuresInfo.isEmailEnabled()); + Assert.assertTrue(featuresInfo.isSmsEnabled()); + Assert.assertTrue(featuresInfo.isTwoFaEnabled()); + Assert.assertTrue(featuresInfo.isNotificationEnabled()); + Assert.assertTrue(featuresInfo.isOauthEnabled()); + } + + + private OAuth2Info createDefaultOAuth2Info() { + return new OAuth2Info(true, Lists.newArrayList( + OAuth2ParamsInfo.builder() + .domainInfos(Lists.newArrayList( + OAuth2DomainInfo.builder().name("domain").scheme(SchemeType.MIXED).build() + )) + .mobileInfos(Collections.emptyList()) + .clientRegistrations(Lists.newArrayList( + validRegistrationInfo() + )) + .build() + )); + } + + private OAuth2RegistrationInfo validRegistrationInfo() { + return OAuth2RegistrationInfo.builder() + .clientId(UUID.randomUUID().toString()) + .clientSecret(UUID.randomUUID().toString()) + .authorizationUri(UUID.randomUUID().toString()) + .accessTokenUri(UUID.randomUUID().toString()) + .scope(Arrays.asList(UUID.randomUUID().toString(), UUID.randomUUID().toString())) + .platforms(Collections.emptyList()) + .userInfoUri(UUID.randomUUID().toString()) + .userNameAttributeName(UUID.randomUUID().toString()) + .jwkSetUri(UUID.randomUUID().toString()) + .clientAuthenticationMethod(UUID.randomUUID().toString()) + .loginButtonLabel(UUID.randomUUID().toString()) + .mapperConfig( + OAuth2MapperConfig.builder() + .type(MapperType.CUSTOM) + .custom( + OAuth2CustomMapperConfig.builder() + .url(UUID.randomUUID().toString()) + .build() + ) + .build() + ) + .build(); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/FeaturesInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/FeaturesInfo.java new file mode 100644 index 0000000000..480e29a6a4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/FeaturesInfo.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.Data; + +@Data +public class FeaturesInfo { + boolean isEmailEnabled; + boolean isSmsEnabled; + boolean isNotificationEnabled; + boolean isOauthEnabled; + boolean isTwoFaEnabled; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index e95fee79ae..0a75e07bd1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -94,7 +94,7 @@ public abstract class DataValidator> { } } - private static boolean doValidateEmail(String email) { + public static boolean doValidateEmail(String email) { if (email == null) { return false; } From a35b3cd6ade41a94ea3bbafc3f13e54de6449ec3 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 23 Mar 2023 21:54:14 +0100 Subject: [PATCH 15/16] System info improvements --- .../server/service/system/DefaultSystemInfoService.java | 3 ++- .../thingsboard/server/controller/BaseHomePageApiTest.java | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index e6f98af4a1..35db65bab2 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -94,8 +94,9 @@ public class DefaultSystemInfoService extends TbApplicationEventListener pageData = update.getData(); From 43efbbe8b90879f3c151f1c76758b10a3cf1b16a Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 24 Mar 2023 17:29:09 +0100 Subject: [PATCH 16/16] fixed testSystemInfoTimeSeriesWsCmd --- .../org/thingsboard/server/controller/BaseHomePageApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java index bcfee283b9..b3032399c2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseHomePageApiTest.java @@ -249,7 +249,7 @@ public abstract class BaseHomePageApiTest extends AbstractControllerTest { Assert.assertEquals(7, listData.get(0).getTimeseries().size()); for (TsValue[] tsv : listData.get(0).getTimeseries().values()) { - Assert.assertEquals(1, tsv.length); + Assert.assertTrue(tsv.length > 1); } }