Merge pull request #1091 from thingsboard/feature/hybrid-db

Feature/hybrid db

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -24,9 +24,10 @@ import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 import org.thingsboard.server.service.component.ComponentDiscoveryService;
 import org.thingsboard.server.service.install.DataUpdateService;
-import org.thingsboard.server.service.install.DatabaseSchemaService;
 import org.thingsboard.server.service.install.DatabaseUpgradeService;
+import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
 import org.thingsboard.server.service.install.SystemDataLoaderService;
+import org.thingsboard.server.service.install.TsDatabaseSchemaService;
 
 @Service
 @Profile("install")
@@ -43,7 +44,10 @@ public class ThingsboardInstallService {
     private Boolean loadDemo;
 
     @Autowired
-    private DatabaseSchemaService databaseSchemaService;
+    private EntityDatabaseSchemaService entityDatabaseSchemaService;
+
+    @Autowired
+    private TsDatabaseSchemaService tsDatabaseSchemaService;
 
     @Autowired
     private DatabaseUpgradeService databaseUpgradeService;
@@ -114,9 +118,13 @@ public class ThingsboardInstallService {
 
                 log.info("Starting ThingsBoard Installation...");
 
-                log.info("Installing DataBase schema...");
+                log.info("Installing DataBase schema for entities...");
+
+                entityDatabaseSchemaService.createDatabaseSchema();
+
+                log.info("Installing DataBase schema for timeseries...");
 
-                databaseSchemaService.createDatabaseSchema();
+                tsDatabaseSchemaService.createDatabaseSchema();
 
                 log.info("Loading system data...");
 

application/src/main/java/org/thingsboard/server/service/install/CassandraDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/CassandraAbstractDatabaseSchemaService.java

@@ -17,24 +17,17 @@ package org.thingsboard.server.service.install;
 
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Profile;
-import org.springframework.stereotype.Service;
 import org.thingsboard.server.dao.cassandra.CassandraInstallCluster;
-import org.thingsboard.server.dao.util.NoSqlDao;
 import org.thingsboard.server.service.install.cql.CQLStatementsParser;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 
-@Service
-@NoSqlDao
-@Profile("install")
 @Slf4j
-public class CassandraDatabaseSchemaService implements DatabaseSchemaService {
+public abstract class CassandraAbstractDatabaseSchemaService implements DatabaseSchemaService {
 
     private static final String CASSANDRA_DIR = "cassandra";
-    private static final String SCHEMA_CQL = "schema.cql";
 
     @Autowired
     private CassandraInstallCluster cluster;
@@ -42,10 +35,16 @@ public class CassandraDatabaseSchemaService implements DatabaseSchemaService {
     @Autowired
     private InstallScripts installScripts;
 
+    private final String schemaCql;
+
+    protected CassandraAbstractDatabaseSchemaService(String schemaCql) {
+        this.schemaCql = schemaCql;
+    }
+
     @Override
     public void createDatabaseSchema() throws Exception {
-        log.info("Installing Cassandra DataBase schema...");
+        log.info("Installing Cassandra DataBase schema part: " + schemaCql);
-        Path schemaFile = Paths.get(installScripts.getDataDir(), CASSANDRA_DIR, SCHEMA_CQL);
+        Path schemaFile = Paths.get(installScripts.getDataDir(), CASSANDRA_DIR, schemaCql);
         loadCql(schemaFile);
 
     }

application/src/main/java/org/thingsboard/server/service/install/CassandraEntityDatabaseSchemaService.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.dao.util.NoSqlDao;
+
+@Service
+@NoSqlDao
+@Profile("install")
+public class CassandraEntityDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService
+        implements EntityDatabaseSchemaService {
+    public CassandraEntityDatabaseSchemaService() {
+        super("schema-entities.cql");
+    }
+}

application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseSchemaService.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.dao.util.NoSqlTsDao;
+
+@Service
+@NoSqlTsDao
+@Profile("install")
+public class CassandraTsDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService
+        implements TsDatabaseSchemaService {
+    public CassandraTsDatabaseSchemaService() {
+        super("schema-ts.cql");
+    }
+}

application/src/main/java/org/thingsboard/server/service/install/EntityDatabaseSchemaService.java

@@ -0,0 +1,19 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+public interface EntityDatabaseSchemaService extends DatabaseSchemaService {
+}

application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/SqlAbstractDatabaseSchemaService.java

@@ -18,9 +18,6 @@ package org.thingsboard.server.service.install;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.context.annotation.Profile;
-import org.springframework.stereotype.Service;
-import org.thingsboard.server.dao.util.SqlDao;
 
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -29,14 +26,10 @@ import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DriverManager;
 
-@Service
-@Profile("install")
 @Slf4j
-@SqlDao
+public abstract class SqlAbstractDatabaseSchemaService implements DatabaseSchemaService {
-public class SqlDatabaseSchemaService implements DatabaseSchemaService {
 
     private static final String SQL_DIR = "sql";
-    private static final String SCHEMA_SQL = "schema.sql";
 
     @Value("${spring.datasource.url}")
     private String dbUrl;
@@ -50,12 +43,18 @@ public class SqlDatabaseSchemaService implements DatabaseSchemaService {
     @Autowired
     private InstallScripts installScripts;
 
+    private final String schemaSql;
+
+    protected SqlAbstractDatabaseSchemaService(String schemaSql) {
+        this.schemaSql = schemaSql;
+    }
+
     @Override
     public void createDatabaseSchema() throws Exception {
 
-        log.info("Installing SQL DataBase schema...");
+        log.info("Installing SQL DataBase schema part: " + schemaSql);
 
-        Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, SCHEMA_SQL);
+        Path schemaFile = Paths.get(installScripts.getDataDir(), SQL_DIR, schemaSql);
         try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
             String sql = new String(Files.readAllBytes(schemaFile), Charset.forName("UTF-8"));
             conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to load initial thingsboard database schema

application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.dao.util.SqlDao;
+
+@Service
+@SqlDao
+@Profile("install")
+public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
+        implements EntityDatabaseSchemaService {
+    public SqlEntityDatabaseSchemaService() {
+        super("schema-entities.sql");
+    }
+}

application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java

@@ -0,0 +1,30 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.dao.util.SqlTsDao;
+
+@Service
+@SqlTsDao
+@Profile("install")
+public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
+        implements TsDatabaseSchemaService {
+    public SqlTsDatabaseSchemaService() {
+        super("schema-ts.sql");
+    }
+}

application/src/main/java/org/thingsboard/server/service/install/TsDatabaseSchemaService.java

@@ -0,0 +1,19 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.install;
+
+public interface TsDatabaseSchemaService extends DatabaseSchemaService {
+}

application/src/main/resources/thingsboard.yml

@@ -159,7 +159,11 @@ quota:
         intervalMin: 2
 
 database:
-  type: "${DATABASE_TYPE:sql}" # cassandra OR sql
+  entities:
+    type: "${DATABASE_TS_TYPE:sql}" # cassandra OR sql
+  ts:
+    type: "${DATABASE_CASSANDRA_TYPE:sql}" # cassandra OR sql (for hybrid mode, only this value should be cassandra)
+
 
 # Cassandra driver configuration parameters
 cassandra:
@@ -206,7 +210,7 @@ cassandra:
     write_consistency_level: "${CASSANDRA_WRITE_CONSISTENCY_LEVEL:ONE}"
     default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}"
     # Specify partitioning size for timestamp key-value storage. Example MINUTES, HOURS, DAYS, MONTHS,INDEFINITE
-    ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
+    ts_key_value_partitioning: "${TS_KV_PARTITIONING:INDEFINITE}"
     ts_key_value_ttl: "${TS_KV_TTL:0}"
     buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}"
     concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}"

application/src/test/java/org/thingsboard/server/controller/ControllerNoSqlTestSuite.java

@@ -32,7 +32,8 @@ public class ControllerNoSqlTestSuite {
     public static CustomCassandraCQLUnit cassandraUnit =
             new CustomCassandraCQLUnit(
                     Arrays.asList(
-                            new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
                     "cassandra-test.yaml", 30000l);

application/src/test/java/org/thingsboard/server/controller/ControllerSqlTestSuite.java

@@ -30,7 +30,7 @@ public class ControllerSqlTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties");
 }

application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java

@@ -32,7 +32,8 @@ public class MqttNoSqlTestSuite {
     public static CustomCassandraCQLUnit cassandraUnit =
             new CustomCassandraCQLUnit(
                     Arrays.asList(
-                            new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
                     "cassandra-test.yaml", 30000l);
 }

application/src/test/java/org/thingsboard/server/mqtt/MqttSqlTestSuite.java

@@ -15,11 +15,9 @@
  */
 package org.thingsboard.server.mqtt;
 
-import org.cassandraunit.dataset.cql.ClassPathCQLDataSet;
 import org.junit.ClassRule;
 import org.junit.extensions.cpsuite.ClasspathSuite;
 import org.junit.runner.RunWith;
-import org.thingsboard.server.dao.CustomCassandraCQLUnit;
 import org.thingsboard.server.dao.CustomSqlUnit;
 
 import java.util.Arrays;
@@ -31,7 +29,7 @@ public class MqttSqlTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties");
 }

application/src/test/java/org/thingsboard/server/rules/RuleEngineNoSqlTestSuite.java

@@ -35,7 +35,8 @@ public class RuleEngineNoSqlTestSuite {
     public static CustomCassandraCQLUnit cassandraUnit =
             new CustomCassandraCQLUnit(
                     Arrays.asList(
-                            new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
                     "cassandra-test.yaml", 30000l);
 

application/src/test/java/org/thingsboard/server/rules/RuleEngineSqlTestSuite.java

@@ -30,7 +30,7 @@ public class RuleEngineSqlTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties");
 }

application/src/test/java/org/thingsboard/server/system/SystemNoSqlTestSuite.java

@@ -34,7 +34,8 @@ public class SystemNoSqlTestSuite {
     public static CustomCassandraCQLUnit cassandraUnit =
             new CustomCassandraCQLUnit(
                     Arrays.asList(
-                            new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-data.cql", false, false)),
                     "cassandra-test.yaml", 30000l);
 }

application/src/test/java/org/thingsboard/server/system/SystemSqlTestSuite.java

@@ -31,7 +31,7 @@ public class SystemSqlTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties");
 

dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraCluster.java

@@ -17,12 +17,12 @@ package org.thingsboard.server.dao.cassandra;
 
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
-import org.thingsboard.server.dao.util.NoSqlDao;
+import org.thingsboard.server.dao.util.NoSqlAnyDao;
 
 import javax.annotation.PostConstruct;
 
 @Component
-@NoSqlDao
+@NoSqlAnyDao
 public class CassandraCluster extends AbstractCassandraCluster {
 
     @Value("${cassandra.keyspace_name}")

dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraInstallCluster.java

@@ -17,12 +17,12 @@ package org.thingsboard.server.dao.cassandra;
 
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Component;
-import org.thingsboard.server.dao.util.NoSqlDao;
+import org.thingsboard.server.dao.util.NoSqlAnyDao;
 
 import javax.annotation.PostConstruct;
 
 @Component
-@NoSqlDao
+@NoSqlAnyDao
 @Profile("install")
 public class CassandraInstallCluster extends AbstractCassandraCluster {
 

dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraQueryOptions.java

@@ -21,14 +21,14 @@ import lombok.Data;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;
-import org.thingsboard.server.dao.util.NoSqlDao;
+import org.thingsboard.server.dao.util.NoSqlAnyDao;
 
 import javax.annotation.PostConstruct;
 
 @Component
 @Configuration
 @Data
-@NoSqlDao
+@NoSqlAnyDao
 public class CassandraQueryOptions {
 
     @Value("${cassandra.query.default_fetch_size}")

dao/src/main/java/org/thingsboard/server/dao/cassandra/CassandraSocketOptions.java

@@ -20,14 +20,14 @@ import lombok.Data;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;
-import org.thingsboard.server.dao.util.NoSqlDao;
+import org.thingsboard.server.dao.util.NoSqlAnyDao;
 
 import javax.annotation.PostConstruct;
 
 @Component
 @Configuration
 @Data
-@NoSqlDao
+@NoSqlAnyDao
 public class CassandraSocketOptions {
 
     @Value("${cassandra.socket.connect_timeout}")

dao/src/main/java/org/thingsboard/server/dao/sql/timeseries/JpaTimeseriesDao.java

@@ -44,6 +44,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
 import org.thingsboard.server.dao.timeseries.TimeseriesDao;
 import org.thingsboard.server.dao.timeseries.TsInsertExecutorType;
 import org.thingsboard.server.dao.util.SqlDao;
+import org.thingsboard.server.dao.util.SqlTsDao;
 
 import javax.annotation.Nullable;
 import javax.annotation.PostConstruct;
@@ -60,7 +61,7 @@ import static org.thingsboard.server.common.data.UUIDConverter.fromTimeUUID;
 
 @Component
 @Slf4j
-@SqlDao
+@SqlTsDao
 public class JpaTimeseriesDao extends JpaAbstractDaoListeningExecutorService implements TimeseriesDao {
 
     @Value("${sql.ts_inserts_executor_type}")

dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry;
 import org.thingsboard.server.dao.model.ModelConstants;
 import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao;
 import org.thingsboard.server.dao.util.NoSqlDao;
+import org.thingsboard.server.dao.util.NoSqlTsDao;
 
 import javax.annotation.Nullable;
 import javax.annotation.PostConstruct;
@@ -70,7 +71,7 @@ import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
  */
 @Component
 @Slf4j
-@NoSqlDao
+@NoSqlTsDao
 public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implements TimeseriesDao {
 
     private static final int MIN_AGGREGATION_STEP_MS = 1000;

dao/src/main/java/org/thingsboard/server/dao/util/BufferedRateLimiter.java

@@ -34,7 +34,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 
 @Component
 @Slf4j
-@NoSqlDao
+@NoSqlAnyDao
 public class BufferedRateLimiter implements AsyncRateLimiter {
 
     private final ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(10));

dao/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java

@@ -0,0 +1,22 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.dao.util;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+
+@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'")
+public @interface NoSqlAnyDao {
+}

dao/src/main/java/org/thingsboard/server/dao/util/NoSqlDao.java

@@ -17,6 +17,6 @@ package org.thingsboard.server.dao.util;
 
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 
-@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "cassandra")
+@ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "cassandra")
 public @interface NoSqlDao {
 }

dao/src/main/java/org/thingsboard/server/dao/util/NoSqlTsDao.java

@@ -0,0 +1,22 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.dao.util;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+
+@ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "cassandra")
+public @interface NoSqlTsDao {
+}

dao/src/main/java/org/thingsboard/server/dao/util/SqlDao.java

@@ -17,6 +17,6 @@ package org.thingsboard.server.dao.util;
 
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 
-@ConditionalOnProperty(prefix = "database", value = "type", havingValue = "sql")
+@ConditionalOnProperty(prefix = "database.entities", value = "type", havingValue = "sql")
 public @interface SqlDao {
 }

dao/src/main/java/org/thingsboard/server/dao/util/SqlTsDao.java

@@ -0,0 +1,22 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.dao.util;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+
+@ConditionalOnProperty(prefix = "database.ts", value = "type", havingValue = "sql")
+public @interface SqlTsDao {
+}

dao/src/main/resources/cassandra/schema.cql → dao/src/main/resources/cassandra/schema-entities.cql

@@ -378,41 +378,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS thingsboard.dashboard_by_tenant_and_searc
 	PRIMARY KEY ( tenant_id, search_text, id )
 	WITH CLUSTERING ORDER BY ( search_text ASC, id DESC );
 
-CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf (
-    entity_type text, // (DEVICE, CUSTOMER, TENANT)
-    entity_id timeuuid,
-    key text,
-    partition bigint,
-    ts bigint,
-    bool_v boolean,
-    str_v text,
-    long_v bigint,
-    dbl_v double,
-    PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)
-);
-
-CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf (
-    entity_type text, // (DEVICE, CUSTOMER, TENANT)
-    entity_id timeuuid,
-    key text,
-    partition bigint,
-    PRIMARY KEY (( entity_type, entity_id, key ), partition)
-) WITH CLUSTERING ORDER BY ( partition ASC )
-  AND compaction = { 'class' :  'LeveledCompactionStrategy'  };
-
-CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (
-    entity_type text, // (DEVICE, CUSTOMER, TENANT)
-    entity_id timeuuid,
-    key text,
-    ts bigint,
-    bool_v boolean,
-    str_v text,
-    long_v bigint,
-    dbl_v double,
-    PRIMARY KEY (( entity_type, entity_id ), key)
-) WITH compaction = { 'class' :  'LeveledCompactionStrategy'  };
-
-
 CREATE TABLE IF NOT EXISTS thingsboard.attributes_kv_cf (
     entity_type text, // (DEVICE, CUSTOMER, TENANT)
     entity_id timeuuid,

dao/src/main/resources/cassandra/schema-ts.cql

@@ -0,0 +1,55 @@
+--
+-- Copyright © 2016-2018 The Thingsboard Authors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE KEYSPACE IF NOT EXISTS thingsboard
+WITH replication = {
+	'class' : 'SimpleStrategy',
+	'replication_factor' : 1
+};
+
+CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_cf (
+    entity_type text, // (DEVICE, CUSTOMER, TENANT)
+    entity_id timeuuid,
+    key text,
+    partition bigint,
+    ts bigint,
+    bool_v boolean,
+    str_v text,
+    long_v bigint,
+    dbl_v double,
+    PRIMARY KEY (( entity_type, entity_id, key, partition ), ts)
+);
+
+CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf (
+    entity_type text, // (DEVICE, CUSTOMER, TENANT)
+    entity_id timeuuid,
+    key text,
+    partition bigint,
+    PRIMARY KEY (( entity_type, entity_id, key ), partition)
+) WITH CLUSTERING ORDER BY ( partition ASC )
+  AND compaction = { 'class' :  'LeveledCompactionStrategy'  };
+
+CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf (
+    entity_type text, // (DEVICE, CUSTOMER, TENANT)
+    entity_id timeuuid,
+    key text,
+    ts bigint,
+    bool_v boolean,
+    str_v text,
+    long_v bigint,
+    dbl_v double,
+    PRIMARY KEY (( entity_type, entity_id ), key)
+) WITH compaction = { 'class' :  'LeveledCompactionStrategy'  };

dao/src/main/resources/sql/schema.sql → dao/src/main/resources/sql/schema-entities.sql

@@ -179,30 +179,6 @@ CREATE TABLE IF NOT EXISTS tenant (
     zip varchar(255)
 );
 
-CREATE TABLE IF NOT EXISTS ts_kv (
-    entity_type varchar(255) NOT NULL,
-    entity_id varchar(31) NOT NULL,
-    key varchar(255) NOT NULL,
-    ts bigint NOT NULL,
-    bool_v boolean,
-    str_v varchar(10000000),
-    long_v bigint,
-    dbl_v double precision,
-    CONSTRAINT ts_kv_unq_key UNIQUE (entity_type, entity_id, key, ts)
-);
-
-CREATE TABLE IF NOT EXISTS ts_kv_latest (
-    entity_type varchar(255) NOT NULL,
-    entity_id varchar(31) NOT NULL,
-    key varchar(255) NOT NULL,
-    ts bigint NOT NULL,
-    bool_v boolean,
-    str_v varchar(10000000),
-    long_v bigint,
-    dbl_v double precision,
-    CONSTRAINT ts_kv_latest_unq_key UNIQUE (entity_type, entity_id, key)
-);
-
 CREATE TABLE IF NOT EXISTS user_credentials (
     id varchar(31) NOT NULL CONSTRAINT user_credentials_pkey PRIMARY KEY,
     activate_token varchar(255) UNIQUE,

dao/src/main/resources/sql/schema-ts.sql

@@ -0,0 +1,39 @@
+--
+-- Copyright © 2016-2018 The Thingsboard Authors
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE TABLE IF NOT EXISTS ts_kv (
+    entity_type varchar(255) NOT NULL,
+    entity_id varchar(31) NOT NULL,
+    key varchar(255) NOT NULL,
+    ts bigint NOT NULL,
+    bool_v boolean,
+    str_v varchar(10000000),
+    long_v bigint,
+    dbl_v double precision,
+    CONSTRAINT ts_kv_unq_key UNIQUE (entity_type, entity_id, key, ts)
+);
+
+CREATE TABLE IF NOT EXISTS ts_kv_latest (
+    entity_type varchar(255) NOT NULL,
+    entity_id varchar(31) NOT NULL,
+    key varchar(255) NOT NULL,
+    ts bigint NOT NULL,
+    bool_v boolean,
+    str_v varchar(10000000),
+    long_v bigint,
+    dbl_v double precision,
+    CONSTRAINT ts_kv_latest_unq_key UNIQUE (entity_type, entity_id, key)
+);

dao/src/test/java/org/thingsboard/server/dao/JpaDaoTestSuite.java

@@ -30,7 +30,7 @@ public class JpaDaoTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties"
     );

dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java

@@ -34,7 +34,9 @@ public class NoSqlDaoServiceTestSuite {
     @ClassRule
     public static CustomCassandraCQLUnit cassandraUnit =
             new CustomCassandraCQLUnit(
-                    Arrays.asList(new ClassPathCQLDataSet("cassandra/schema.cql", false, false),
+                    Arrays.asList(
+                            new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false),
+                            new ClassPathCQLDataSet("cassandra/schema-entities.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-data.cql", false, false),
                             new ClassPathCQLDataSet("cassandra/system-test.cql", false, false)),
                     "cassandra-test.yaml", 30000L);

dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java

@@ -30,7 +30,7 @@ public class SqlDaoServiceTestSuite {
 
     @ClassRule
     public static CustomSqlUnit sqlUnit = new CustomSqlUnit(
-            Arrays.asList("sql/schema.sql", "sql/system-data.sql", "sql/system-test.sql"),
+            Arrays.asList("sql/schema-ts.sql", "sql/schema-entities.sql", "sql/system-data.sql", "sql/system-test.sql"),
             "sql/drop-all-tables.sql",
             "sql-test.properties"
     );

dao/src/test/resources/nosql-test.properties

@@ -1,4 +1,5 @@
-database.type=cassandra
+database.entities.type=cassandra
+database.ts.type=cassandra
 
 cassandra.queue.partitioning=HOURS
 cassandra.queue.ack.ttl=3600

dao/src/test/resources/sql-test.properties

@@ -1,4 +1,5 @@
-database.type=sql
+database.ts.type=sql
+database.entities.type=sql
 
 sql.ts_inserts_executor_type=fixed
 sql.ts_inserts_fixed_thread_pool_size=10

docker/k8s/cassandra-setup.yaml

@@ -30,7 +30,9 @@ spec:
       value: "cassandra-headless"
     - name : CASSANDRA_PORT
       value: "9042"
-    - name : DATABASE_TYPE
+    - name : DATABASE_ENTITIES_TYPE
+      value: "cassandra"
+    - name : DATABASE_TS_TYPE
       value: "cassandra"
     - name : CASSANDRA_URL
       value: "cassandra-headless:9042"

docker/k8s/cassandra-upgrade.yaml

@@ -30,7 +30,9 @@ spec:
       value: "cassandra-headless"
     - name : CASSANDRA_PORT
       value: "9042"
-    - name : DATABASE_TYPE
+    - name : DATABASE_ENTITIES_TYPE
+      value: "cassandra"
+    - name : DATABASE_TS_TYPE
       value: "cassandra"
     - name : CASSANDRA_URL
       value: "cassandra-headless:9042"

docker/k8s/tb.yaml

@@ -120,7 +120,12 @@ spec:
             configMapKeyRef:
               name: tb-config
               key: cassandra.url
-        - name: DATABASE_TYPE
+        - name: DATABASE_ENTITIES_TYPE
+          valueFrom:
+            configMapKeyRef:
+              name: tb-config
+              key: database.type
+        - name: DATABASE_TS_TYPE
           valueFrom:
             configMapKeyRef:
               name: tb-config

docker/tb.env

@@ -8,7 +8,8 @@ COAP_BIND_PORT=5683
 ZOOKEEPER_URL=zk:2181
 
 # type of database to use: sql[DEFAULT] or cassandra
-DATABASE_TYPE=sql
+DATABASE_TS_TYPE=sql
+DATABASE_ENTITIES_TYPE=sql
 
 # cassandra db config
 CASSANDRA_URL=cassandra:9042

docker/tb/run-application.sh

@@ -23,7 +23,7 @@ printenv | awk -F "=" '{print "export " $1 "='\''" $2 "'\''"}' >> /usr/share/thi
 
 cat /usr/share/thingsboard/conf/thingsboard.conf
 
-if [ "$DATABASE_TYPE" == "cassandra" ]; then
+if [ "$DATABASE_ENTITIES_TYPE" == "cassandra" ]; then
     until nmap $CASSANDRA_HOST -p $CASSANDRA_PORT | grep "$CASSANDRA_PORT/tcp open\|filtered"
     do
       echo "Wait for cassandra db to start..."
@@ -31,7 +31,7 @@ if [ "$DATABASE_TYPE" == "cassandra" ]; then
     done
 fi
 
-if [ "$DATABASE_TYPE" == "sql" ]; then
+if [ "$DATABASE_ENTITIES_TYPE" == "sql" ]; then
     if [ "$SPRING_DRIVER_CLASS_NAME" == "org.postgresql.Driver" ]; then
         until nmap $POSTGRES_HOST -p $POSTGRES_PORT | grep "$POSTGRES_PORT/tcp open"
         do