Merge with master

application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql

@@ -64,6 +64,7 @@ BEGIN
                                AND tablename like 'ts_kv_' || '%'
                                AND tablename != 'ts_kv_latest'
                                AND tablename != 'ts_kv_dictionary'
+                               AND tablename != 'ts_kv_indefinite'
                 LOOP
                     IF partition != partition_by_max_ttl_date THEN
                         IF partition_year IS NOT NULL THEN

application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql

@@ -59,8 +59,8 @@ $$
 DECLARE
     tenant_cursor CURSOR FOR select tenant.id as tenant_id
                              from tenant;
-    tenant_id_record     varchar;
+    tenant_id_record     uuid;
-    customer_id_record   varchar;
+    customer_id_record   uuid;
     tenant_ttl           bigint;
     customer_ttl         bigint;
     deleted_for_entities bigint;

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -176,8 +176,13 @@ public class ThingsboardInstallService {
                             log.info("Upgrading ThingsBoard from version 3.1.0 to 3.1.1 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.1.0");
                         case "3.1.1":
-                            log.info("Upgrading ThingsBoard from version 3.1.1 to 3.2.0 ...");
+                            log.info("Upgrading ThingsBoard from version 3.1.1 to 3.1.2 ...");
-                            databaseEntitiesUpgradeService.upgradeDatabase("3.1.1");
+                            if (databaseTsUpgradeService != null) {
+                                databaseTsUpgradeService.upgradeDatabase("3.1.1");
+                            }
+                        case "3.1.2":
+                            log.info("Upgrading ThingsBoard from version 3.1.2 to 3.2.0 ...");
+                            databaseEntitiesUpgradeService.upgradeDatabase("3.1.2");
                             log.info("Updating system data...");
                             systemDataLoaderService.updateSystemWidgets();
                             break;

application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java

@@ -49,6 +49,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase
                 log.info("Schema updated.");
                 break;
             case "2.5.0":
+            case "3.1.1":
                 break;
             default:
                 throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion);

application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java

@@ -195,6 +195,14 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
                     executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
                 }
                 break;
+            case "3.1.1":
+                try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
+                    log.info("Load TTL functions ...");
+                    loadSql(conn, LOAD_TTL_FUNCTIONS_SQL);
+                    log.info("Load Drop Partitions functions ...");
+                    loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL);
+                }
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@@ -324,7 +324,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                     log.info("Schema updated.");
                 }
                 break;
-            case "3.1.1":
+            case "3.1.2":
                 try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
                     log.info("Updating schema ...");
                     if (isOldSchema(conn, 3001000)) {
@@ -339,7 +339,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                         } catch (Exception e) {
                         }
 
-                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_before.sql");
+                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.2", "schema_update_before.sql");
                         loadSql(schemaUpdateFile, conn);
 
                         log.info("Creating default tenant profiles...");
@@ -371,7 +371,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                         log.info("Updating device profiles...");
                         conn.createStatement().execute("call update_device_profiles()");
 
-                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.1", "schema_update_after.sql");
+                        schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.2", "schema_update_after.sql");
                         loadSql(schemaUpdateFile, conn);
 
                         conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3002000;");

application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@@ -177,6 +177,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
                     executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001");
                 }
                 break;
+            case "3.1.1":
+                break;
             default:
                 throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
         }

application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java

@@ -38,19 +38,15 @@ public abstract class AbstractCleanUpService {
     @Value("${spring.datasource.password}")
     protected String dbPassword;
 
-    protected long executeQuery(Connection conn, String query) {
+    protected long executeQuery(Connection conn, String query) throws SQLException {
-        long removed = 0L;
+        try (Statement statement = conn.createStatement()) {
-        try {
-            Statement statement = conn.createStatement();
             ResultSet resultSet = statement.executeQuery(query);
-            getWarnings(statement);
+            if (log.isDebugEnabled()) {
+                getWarnings(statement);
+            }
             resultSet.next();
-            removed = resultSet.getLong(1);
+            return resultSet.getLong(1);
-            log.debug("Successfully executed query: {}", query);
-        } catch (SQLException e) {
-            log.debug("Failed to execute query: {} due to: {}", query, e.getMessage());
         }
-        return removed;
     }
 
     protected void getWarnings(Statement statement) throws SQLException {
@@ -65,6 +61,6 @@ public abstract class AbstractCleanUpService {
         }
     }
 
-    protected abstract void doCleanUp(Connection connection);
+    protected abstract void doCleanUp(Connection connection) throws SQLException;
 
 }

application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java

@@ -52,7 +52,7 @@ public class EventsCleanUpService extends AbstractCleanUpService {
     }
 
     @Override
-    protected void doCleanUp(Connection connection) {
+    protected void doCleanUp(Connection connection) throws SQLException {
         long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);");
         log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
     }

application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java

@@ -23,6 +23,7 @@ import org.thingsboard.server.dao.util.PsqlDao;
 import org.thingsboard.server.dao.util.SqlTsDao;
 
 import java.sql.Connection;
+import java.sql.SQLException;
 
 @SqlTsDao
 @PsqlDao
@@ -34,10 +35,10 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi
     private String partitionType;
 
     @Override
-    protected void doCleanUp(Connection connection) {
+    protected void doCleanUp(Connection connection) throws SQLException {
-        long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
+            long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);");
-        log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
+            log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved);
-        long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);");
+            long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);");
-        log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
+            log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
     }
 }

application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java

@@ -21,6 +21,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
 import org.thingsboard.server.dao.util.TimescaleDBTsDao;
 
 import java.sql.Connection;
+import java.sql.SQLException;
 
 @TimescaleDBTsDao
 @Service
@@ -28,7 +29,7 @@ import java.sql.Connection;
 public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService {
 
     @Override
-    protected void doCleanUp(Connection connection) {
+    protected void doCleanUp(Connection connection) throws SQLException {
         long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);");
         log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved);
     }

application/src/main/resources/thingsboard.yml

@@ -194,8 +194,21 @@ cassandra:
   url: "${CASSANDRA_URL:127.0.0.1:9042}"
   # Specify local datacenter name
   local_datacenter: "${CASSANDRA_LOCAL_DATACENTER:datacenter1}"
-  # Enable/disable secure connection
+  ssl:
-  ssl: "${CASSANDRA_USE_SSL:false}"
+    # Enable/disable secure connection
+    enabled: "${CASSANDRA_USE_SSL:false}"
+    # Enable/disable validation of Cassandra server hostname
+    # If enabled, hostname of Cassandra server must match CN of server certificate
+    hostname_validation: "${CASSANDRA_SSL_HOSTNAME_VALIDATION:true}"
+    # Set trust store for client authentication of server (optional, uses trust store from default SSLContext if not set)
+    trust_store: "${CASSANDRA_SSL_TRUST_STORE:}"
+    trust_store_password: "${CASSANDRA_SSL_TRUST_STORE_PASSWORD:}"
+    # Set key store for server authentication of client (optional, uses key store from default SSLContext if not set)
+    # A key store is only needed if the Cassandra server requires client authentication
+    key_store: "${CASSANDRA_SSL_KEY_STORE:}"
+    key_store_password: "${CASSANDRA_SSL_KEY_STORE_PASSWORD:}"
+    # Comma separated list of cipher suites (optional, uses Java default cipher suites if not set)
+    cipher_suites: "${CASSANDRA_SSL_CIPHER_SUITES:}"
   # Enable/disable JMX
   jmx: "${CASSANDRA_USE_JMX:false}"
   # Enable/disable metrics collection.

common/dao-api/src/main/java/org/thingsboard/server/dao/cassandra/CassandraDriverOptions.java

@@ -80,8 +80,22 @@ public class CassandraDriverOptions {
 
     @Value("${cassandra.compression}")
     private String compression;
-    @Value("${cassandra.ssl}")
+    
+    @Value("${cassandra.ssl.enabled}")
     private Boolean ssl;
+    @Value("${cassandra.ssl.key_store}")
+    private String sslKeyStore;
+    @Value("${cassandra.ssl.key_store_password}")
+    private String sslKeyStorePassword;
+    @Value("${cassandra.ssl.trust_store}")
+    private String sslTrustStore;
+    @Value("${cassandra.ssl.trust_store_password}")
+    private String sslTrustStorePassword;
+    @Value("${cassandra.ssl.hostname_validation}")
+    private Boolean sslHostnameValidation;
+    @Value("${cassandra.ssl.cipher_suites}")
+    private List<String> sslCipherSuites;
+    
     @Value("${cassandra.metrics}")
     private Boolean metrics;
 
@@ -120,7 +134,19 @@ public class CassandraDriverOptions {
 
         if (this.ssl) {
             driverConfigBuilder.withString(DefaultDriverOption.SSL_ENGINE_FACTORY_CLASS,
-                    "DefaultSslEngineFactory");
+                    "DefaultSslEngineFactory")
+                .withBoolean(DefaultDriverOption.SSL_HOSTNAME_VALIDATION, this.sslHostnameValidation);
+            if(!this.sslTrustStore.isEmpty()) {
+                driverConfigBuilder.withString(DefaultDriverOption.SSL_TRUSTSTORE_PATH, this.sslTrustStore)
+                    .withString(DefaultDriverOption.SSL_TRUSTSTORE_PASSWORD, this.sslTrustStorePassword);
+            }
+            if(!this.sslKeyStore.isEmpty()) {
+                driverConfigBuilder.withString(DefaultDriverOption.SSL_KEYSTORE_PATH, this.sslKeyStore)
+                    .withString(DefaultDriverOption.SSL_KEYSTORE_PASSWORD, this.sslKeyStorePassword);
+            }
+            if(!this.sslCipherSuites.isEmpty()) {
+                driverConfigBuilder.withStringList(DefaultDriverOption.SSL_CIPHER_SUITES, this.sslCipherSuites);
+            }
         }
 
         if (this.metrics) {

dao/src/main/resources/sql/schema-ts-psql.sql

@@ -84,6 +84,7 @@ BEGIN
                                AND tablename like 'ts_kv_' || '%'
                                AND tablename != 'ts_kv_latest'
                                AND tablename != 'ts_kv_dictionary'
+                               AND tablename != 'ts_kv_indefinite'
                 LOOP
                     IF partition != partition_by_max_ttl_date THEN
                         IF partition_year IS NOT NULL THEN
@@ -187,8 +188,8 @@ $$
 DECLARE
     tenant_cursor CURSOR FOR select tenant.id as tenant_id
                              from tenant;
-    tenant_id_record     varchar;
+    tenant_id_record     uuid;
-    customer_id_record   varchar;
+    customer_id_record   uuid;
     tenant_ttl           bigint;
     customer_ttl         bigint;
     deleted_for_entities bigint;

dao/src/test/resources/cassandra-test.properties

@@ -6,7 +6,13 @@ cassandra.url=127.0.0.1:9142
 
 cassandra.local_datacenter=datacenter1
 
-cassandra.ssl=false
+cassandra.ssl.enabled=false
+cassandra.ssl.hostname_validation=false
+cassandra.ssl.trust_store=
+cassandra.ssl.trust_store_password=
+cassandra.ssl.key_store=
+cassandra.ssl.key_store_password=
+cassandra.ssl.cipher_suites=
 
 cassandra.jmx=false
 

ui-ngx/src/app/core/api/widget-subscription.ts

@@ -403,7 +403,7 @@ export class WidgetSubscription implements IWidgetSubscription {
         configDatasource: datasource,
         configDatasourceIndex: index,
         dataLoaded: (pageData, data1, datasourceIndex, pageLink) => {
-          this.dataLoaded(pageData, data1, datasourceIndex, pageLink, true)
+          this.dataLoaded(pageData, data1, datasourceIndex, pageLink, true);
         },
         initialPageDataChanged: this.initialPageDataChanged.bind(this),
         dataUpdated: this.dataUpdated.bind(this),
@@ -575,7 +575,7 @@ export class WidgetSubscription implements IWidgetSubscription {
 
   updateDataVisibility(index: number): void {
     if (this.displayLegend) {
-      const hidden = this.legendData.keys[index].dataKey.hidden;
+      const hidden = this.legendData.keys.find(key => key.dataIndex === index).dataKey.hidden;
       if (hidden) {
         this.hiddenData[index].data = this.data[index].data;
         this.data[index].data = [];
@@ -804,7 +804,7 @@ export class WidgetSubscription implements IWidgetSubscription {
         configDatasourceIndex: datasourceIndex,
         subscriptionTimewindow: this.subscriptionTimewindow,
         dataLoaded: (pageData, data1, datasourceIndex1, pageLink1) => {
-          this.dataLoaded(pageData, data1, datasourceIndex1, pageLink1, true)
+          this.dataLoaded(pageData, data1, datasourceIndex1, pageLink1, true);
         },
         dataUpdated: this.dataUpdated.bind(this),
         updateRealtimeSubscription: () => {
@@ -1149,7 +1149,7 @@ export class WidgetSubscription implements IWidgetSubscription {
         this.onSubscriptionMessage({
           severity: 'warn',
           message
-        })
+        });
       }
     }
     if (isUpdate) {
@@ -1274,7 +1274,7 @@ export class WidgetSubscription implements IWidgetSubscription {
     const configuredDatasource = this.configuredDatasources[datasourceIndex];
     const startIndex = configuredDatasource.dataKeyStartIndex;
     const dataKeysCount = configuredDatasource.dataKeys.length;
-    const index = startIndex + dataIndex*dataKeysCount + dataKeyIndex;
+    const index = startIndex + dataIndex * dataKeysCount + dataKeyIndex;
     let update = true;
     let currentData: DataSetHolder;
     if (this.displayLegend && this.legendData.keys[index].dataKey.hidden) {
@@ -1331,7 +1331,7 @@ export class WidgetSubscription implements IWidgetSubscription {
   }
 
   private updateLegend(dataIndex: number, data: DataSet, detectChanges: boolean) {
-    const dataKey = this.legendData.keys[dataIndex].dataKey;
+    const dataKey = this.legendData.keys.find(key => key.dataIndex === dataIndex).dataKey;
     const decimals = isDefined(dataKey.decimals) ? dataKey.decimals : this.decimals;
     const units = dataKey.units && dataKey.units.length ? dataKey.units : this.units;
     const legendKeyData = this.legendData.data[dataIndex];

ui-ngx/src/app/modules/common/modules-map.ts

@@ -60,6 +60,7 @@ import * as AngularMaterialStepper from '@angular/material/stepper';
 import * as AngularMaterialTable from '@angular/material/table';
 import * as AngularMaterialTabs from '@angular/material/tabs';
 import * as AngularMaterialToolbar from '@angular/material/toolbar';
+import * as AngularMaterialTooltip from '@angular/material/tooltip';
 import * as AngularMaterialTree from '@angular/material/tree';
 import * as NgrxStore from '@ngrx/store';
 import * as RxJs from 'rxjs';
@@ -119,6 +120,7 @@ export const modulesMap: {[key: string]: any} = {
   '@angular/material/table': SystemJS.newModule(AngularMaterialTable),
   '@angular/material/tabs': SystemJS.newModule(AngularMaterialTabs),
   '@angular/material/toolbar': SystemJS.newModule(AngularMaterialToolbar),
+  '@angular/material/tooltip': SystemJS.newModule(AngularMaterialTooltip),
   '@angular/material/tree': SystemJS.newModule(AngularMaterialTree),
   '@ngrx/store': SystemJS.newModule(NgrxStore),
   rxjs: SystemJS.newModule(RxJs),

ui-ngx/src/app/modules/home/components/widget/legend.component.html

@@ -32,7 +32,7 @@
         <td><span class="tb-legend-line" [ngStyle]="{backgroundColor: legendKey.dataKey.color}"></span></td>
         <td class="tb-legend-label"
             (click)="toggleHideData(legendKey.dataIndex)"
-            [ngClass]="{ 'tb-hidden-label': legendData.keys[legendKey.dataIndex].dataKey.hidden, 'tb-horizontal': isHorizontal }">
+            [ngClass]="{ 'tb-hidden-label': legendKey.dataKey.hidden, 'tb-horizontal': isHorizontal }">
           {{ legendKey.dataKey.label }}
         </td>
         <td class="tb-legend-value" *ngIf="legendConfig.showMin === true">{{ legendData.data[legendKey.dataIndex].min }}</td>
@@ -47,7 +47,7 @@
         <span class="tb-legend-line" [ngStyle]="{backgroundColor: legendKey.dataKey.color}"></span>
         <span class="tb-legend-label"
               (click)="toggleHideData(legendKey.dataIndex)"
-              [ngClass]="{ 'tb-hidden-label': legendData.keys[legendKey.dataIndex].dataKey.hidden}">
+              [ngClass]="{ 'tb-hidden-label': legendKey.dataKey.hidden}">
                   {{ legendKey.dataKey.label }}
         </span>
       </td>

ui-ngx/src/app/modules/home/components/widget/legend.component.ts

@@ -52,8 +52,9 @@ export class LegendComponent implements OnInit {
   }
 
   toggleHideData(index: number) {
-    if (!this.legendData.keys[index].dataKey.settings.disableDataHiding) {
+    const dataKey = this.legendData.keys.find(key => key.dataIndex === index).dataKey;
-      this.legendData.keys[index].dataKey.hidden = !this.legendData.keys[index].dataKey.hidden;
+    if (!dataKey.settings.disableDataHiding) {
+      dataKey.hidden = !dataKey.hidden;
       this.legendKeyHiddenChange.emit(index);
     }
   }