added new metrics

application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java

@@ -21,22 +21,27 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
 import org.thingsboard.server.common.data.SystemInfo;
 import org.thingsboard.server.common.data.SystemInfoData;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
 import org.thingsboard.server.common.data.kv.DoubleDataEntry;
+import org.thingsboard.server.common.data.kv.JsonDataEntry;
 import org.thingsboard.server.common.data.kv.LongDataEntry;
 import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
 import org.thingsboard.server.queue.discovery.DiscoveryService;
+import org.thingsboard.server.queue.discovery.PartitionService;
+import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
+import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
 
 import javax.annotation.Nullable;
-import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -45,15 +50,19 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import static org.thingsboard.common.util.SystemUtil.getFreeMemory;
 import static org.thingsboard.common.util.SystemUtil.getCpuUsage;
 import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace;
 import static org.thingsboard.common.util.SystemUtil.getMemoryUsage;
+import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage;
+import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace;
+import static org.thingsboard.common.util.SystemUtil.getTotalMemory;
 
 @TbCoreComponent
 @Service
 @RequiredArgsConstructor
 @Slf4j
-public class DefaultSystemInfoService implements SystemInfoService {
+public class DefaultSystemInfoService extends TbApplicationEventListener<PartitionChangeEvent> implements SystemInfoService {
 
     public static final FutureCallback<Integer> CALLBACK = new FutureCallback<>() {
         @Override
@@ -67,6 +76,7 @@ public class DefaultSystemInfoService implements SystemInfoService {
     };
 
     private final TbServiceInfoProvider serviceInfoProvider;
+    private final PartitionService partitionService;
     private final DiscoveryService discoveryService;
     private final TelemetrySubscriptionService telemetryService;
     private ScheduledExecutorService scheduler;
@@ -74,18 +84,15 @@ public class DefaultSystemInfoService implements SystemInfoService {
     @Value("${zk.enabled:false}")
     private boolean zkEnabled;
 
-    @PostConstruct
+    @Override
-    private void init() {
+    protected void onTbApplicationEvent(PartitionChangeEvent partitionChangeEvent) {
-        if (!zkEnabled) {
+        if (ServiceType.TB_CORE.equals(partitionChangeEvent.getServiceType())) {
-            scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler"));
+            if (scheduler == null && partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition()) {
-            scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES);
+                scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("tb-system-info-scheduler"));
-        }
+                scheduler.scheduleAtFixedRate(this::saveCurrentSystemInfo, 0, 1, TimeUnit.MINUTES);
-    }
+            } else {
-
+                destroy();
-    @PreDestroy
+            }
-    private void destroy() {
-        if (scheduler != null) {
-            scheduler.shutdownNow();
         }
     }
 
@@ -96,13 +103,7 @@ public class DefaultSystemInfoService implements SystemInfoService {
         ServiceInfo serviceInfo = serviceInfoProvider.getServiceInfoWithCurrentSystemInfo();
 
         if (zkEnabled) {
-            List<SystemInfoData> clusterSystemData = new ArrayList<>();
+            systemInfo.setSystemData(getSystemData(serviceInfo));
-            clusterSystemData.add(createSystemInfoData(serviceInfo));
-            this.discoveryService.getOtherServers()
-                    .stream()
-                    .map(this::createSystemInfoData)
-                    .forEach(clusterSystemData::add);
-            systemInfo.setSystemData(clusterSystemData);
         } else {
             systemInfo.setMonolith(true);
             systemInfo.setSystemData(Collections.singletonList(createSystemInfoData(serviceInfo)));
@@ -112,6 +113,22 @@ public class DefaultSystemInfoService implements SystemInfoService {
     }
 
     protected void saveCurrentSystemInfo() {
+        if (zkEnabled) {
+            saveCurrentClusterSystemInfo();
+        } else {
+            saveCurrentMonolithSystemInfo();
+        }
+    }
+
+    private void saveCurrentClusterSystemInfo() {
+        long ts = System.currentTimeMillis();
+
+        List<SystemInfoData> clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfoWithCurrentSystemInfo());
+        BasicTsKvEntry clusterDataKv = new BasicTsKvEntry(ts, new JsonDataEntry("clusterSystemData", JacksonUtil.toString(clusterSystemData)));
+        doSave(Collections.singletonList(clusterDataKv));
+    }
+
+    private void saveCurrentMonolithSystemInfo() {
         long ts = System.currentTimeMillis();
         List<TsKvEntry> tsList = new ArrayList<>();
 
@@ -119,16 +136,46 @@ public class DefaultSystemInfoService implements SystemInfoService {
         if (memoryUsage != null) {
             tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", memoryUsage)));
         }
+        Long totalMemory = getTotalMemory();
+        if (totalMemory != null) {
+            tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalMemory", totalMemory)));
+        }
+        Long freeMemory = getFreeMemory();
+        if (freeMemory != null) {
+            tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeMemory", freeMemory)));
+        }
         Double cpuUsage = getCpuUsage();
         if (cpuUsage != null) {
             tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("cpuUsage", cpuUsage)));
         }
+        Double totalCpuUsage = getTotalCpuUsage();
+        if (totalCpuUsage != null) {
+            tsList.add(new BasicTsKvEntry(ts, new DoubleDataEntry("totalCpuUsage", totalCpuUsage)));
+        }
         Long freeDiscSpace = getFreeDiscSpace();
         if (freeDiscSpace != null) {
             tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("freeDiscSpace", freeDiscSpace)));
         }
+        Long totalDiscSpace = getTotalDiscSpace();
+        if (totalDiscSpace != null) {
+            tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("totalDiscSpace", totalDiscSpace)));
+        }
+
+        doSave(tsList);
+    }
 
-        telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, tsList, CALLBACK);
+    private void doSave(List<TsKvEntry> telemetry) {
+        telemetryService.saveAndNotifyInternal(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, telemetry, CALLBACK);
+    }
+
+    private List<SystemInfoData> getSystemData(ServiceInfo serviceInfo) {
+        List<SystemInfoData> clusterSystemData = new ArrayList<>();
+        clusterSystemData.add(createSystemInfoData(serviceInfo));
+        this.discoveryService.getOtherServers()
+                .stream()
+                .map(this::createSystemInfoData)
+                .forEach(clusterSystemData::add);
+        return clusterSystemData;
     }
 
     private SystemInfoData createSystemInfoData(ServiceInfo serviceInfo) {
@@ -136,10 +183,21 @@ public class DefaultSystemInfoService implements SystemInfoService {
         SystemInfoData infoData = new SystemInfoData();
         infoData.setServiceId(serviceInfo.getServiceId());
         infoData.setServiceType(serviceTypes.size() > 1 ? "MONOLITH" : serviceTypes.get(0));
-        infoData.setMemUsage(serviceInfo.getSystemInfo().getMemoryUsage());
+        infoData.setMemoryUsage(serviceInfo.getSystemInfo().getMemoryUsage());
+        infoData.setTotalMemory(serviceInfo.getSystemInfo().getTotalMemory());
+        infoData.setFreeMemory(serviceInfo.getSystemInfo().getFreeMemory());
         infoData.setCpuUsage(serviceInfo.getSystemInfo().getCpuUsage());
+        infoData.setTotalCpuUsage(serviceInfo.getSystemInfo().getTotalCpuUsage());
         infoData.setFreeDiscSpace(serviceInfo.getSystemInfo().getFreeDiscSpace());
+        infoData.setTotalDiscSpace(serviceInfo.getSystemInfo().getTotalDiscSpace());
         return infoData;
     }
 
+    @PreDestroy
+    private void destroy() {
+        if (scheduler != null) {
+            scheduler.shutdownNow();
+            scheduler = null;
+        }
+    }
 }

common/cluster-api/src/main/proto/queue.proto

@@ -31,9 +31,13 @@ message ServiceInfo {
 }
 
 message SystemInfoProto {
-  int64 memoryUsage = 1;
+  double cpuUsage = 1;
-  double cpuUsage = 2;
+  double totalCpuUsage = 2;
-  int64 freeDiscSpace = 3;
+  int64 memoryUsage = 3;
+  int64 totalMemory = 4;
+  int64 freeMemory = 5;
+  int64 freeDiscSpace = 6;
+  int64 totalDiscSpace = 7;
 }
 
 /**

common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java

@@ -28,8 +28,16 @@ public class SystemInfoData {
     private String serviceType;
     @ApiModelProperty(position = 3, value = "CPU usage.")
     private Double cpuUsage;
-    @ApiModelProperty(position = 4, value = "Memory usage.")
+    @ApiModelProperty(position = 4, value = "Total CPU usage.")
-    private Long memUsage;
+    private Double totalCpuUsage;
-    @ApiModelProperty(position = 5, value = "Free disc space.")
+    @ApiModelProperty(position = 5, value = "Memory usage in bytes.")
+    private Long memoryUsage;
+    @ApiModelProperty(position = 6, value = "Total memory in bytes.")
+    private Long totalMemory;
+    @ApiModelProperty(position = 6, value = "Free memory in bytes.")
+    private Long freeMemory;
+    @ApiModelProperty(position = 7, value = "Free disc space in bytes.")
     private Long freeDiscSpace;
+    @ApiModelProperty(position = 7, value = "Total disc space in bytes.")
+    private Long totalDiscSpace;
 }

common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java

@@ -37,9 +37,13 @@ import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
+import static org.thingsboard.common.util.SystemUtil.getFreeMemory;
 import static org.thingsboard.common.util.SystemUtil.getCpuUsage;
 import static org.thingsboard.common.util.SystemUtil.getFreeDiscSpace;
 import static org.thingsboard.common.util.SystemUtil.getMemoryUsage;
+import static org.thingsboard.common.util.SystemUtil.getTotalCpuUsage;
+import static org.thingsboard.common.util.SystemUtil.getTotalDiscSpace;
+import static org.thingsboard.common.util.SystemUtil.getTotalMemory;
 
 @Component
 @Slf4j
@@ -118,14 +122,30 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
         if (memoryUsage != null) {
             builder.setMemoryUsage(memoryUsage);
         }
+        Long totalMemory = getTotalMemory();
+        if (totalMemory != null) {
+            builder.setTotalMemory(totalMemory);
+        }
+        Long freeMemory = getFreeMemory();
+        if (freeMemory != null) {
+            builder.setFreeMemory(freeMemory);
+        }
         Double cpuUsage = getCpuUsage();
         if (cpuUsage != null) {
             builder.setCpuUsage(cpuUsage);
         }
+        Double totalCpuUsage = getTotalCpuUsage();
+        if (totalCpuUsage != null) {
+            builder.setTotalCpuUsage(totalCpuUsage);
+        }
         Long freeDiscSpace = getFreeDiscSpace();
         if (freeDiscSpace != null) {
             builder.setFreeDiscSpace(freeDiscSpace);
         }
+        Long totalDiscSpace = getTotalDiscSpace();
+        if (totalDiscSpace != null) {
+            builder.setTotalDiscSpace(totalDiscSpace);
+        }
 
         return builder.build();
     }

common/util/pom.xml

@@ -88,6 +88,10 @@
             <groupId>org.thingsboard.common</groupId>
             <artifactId>data</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.github.dblock</groupId>
+            <artifactId>oshi-core</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

common/util/src/main/java/org/thingsboard/common/util/SystemUtil.java

@@ -16,10 +16,11 @@
 package org.thingsboard.common.util;
 
 import lombok.extern.slf4j.Slf4j;
+import oshi.SystemInfo;
+import oshi.hardware.HardwareAbstractionLayer;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryMXBean;
-import java.lang.management.OperatingSystemMXBean;
 import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.nio.file.Paths;
@@ -27,6 +28,13 @@ import java.nio.file.Paths;
 @Slf4j
 public class SystemUtil {
 
+    private static final HardwareAbstractionLayer HARDWARE;
+
+    static {
+        SystemInfo si = new SystemInfo();
+        HARDWARE = si.getHardware();
+    }
+
     public static Long getMemoryUsage() {
         try {
             MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
@@ -37,16 +45,42 @@ public class SystemUtil {
         return null;
     }
 
+    public static Long getTotalMemory() {
+        try {
+            return HARDWARE.getMemory().getTotal();
+        } catch (Exception e) {
+            log.debug("Failed to get total memory!!!", e);
+        }
+        return null;
+    }
+
+    public static Long getFreeMemory() {
+        try {
+            return HARDWARE.getMemory().getAvailable();
+        } catch (Exception e) {
+            log.debug("Failed to get free memory!!!", e);
+        }
+        return null;
+    }
+
     public static Double getCpuUsage() {
         try {
-            OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean();
+            return prepare(HARDWARE.getProcessor().getSystemLoadAverage());
-            return osBean.getSystemLoadAverage();
         } catch (Exception e) {
             log.debug("Failed to get cpu usage!!!", e);
         }
         return null;
     }
 
+    public static Double getTotalCpuUsage() {
+        try {
+            return prepare(HARDWARE.getProcessor().getSystemCpuLoad() * 100);
+        } catch (Exception e) {
+            log.debug("Failed to get total cpu usage!!!", e);
+        }
+        return null;
+    }
+
     public static Long getFreeDiscSpace() {
         try {
             FileStore store = Files.getFileStore(Paths.get("/"));
@@ -56,4 +90,18 @@ public class SystemUtil {
         }
         return null;
     }
+
+    public static Long getTotalDiscSpace() {
+        try {
+            FileStore store = Files.getFileStore(Paths.get("/"));
+            return store.getTotalSpace();
+        } catch (Exception e) {
+            log.debug("Failed to get total disc space!!!", e);
+        }
+        return null;
+    }
+
+    private static Double prepare(Double d) {
+        return (int) (d * 100) / 100.0;
+    }
 }

pom.xml

@@ -149,6 +149,7 @@
         <allure-testng.version>2.21.0</allure-testng.version>
         <allure-maven.version>2.12.0</allure-maven.version>
         <slack-api.version>1.12.1</slack-api.version>
+        <oshi.version>3.4.0</oshi.version>
     </properties>
 
     <modules>
@@ -1987,6 +1988,11 @@
                 <artifactId>exp4j</artifactId>
                 <version>${exp4j.version}</version>
             </dependency>
+            <dependency>
+                <groupId>com.github.dblock</groupId>
+                <artifactId>oshi-core</artifactId>
+                <version>${oshi.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>