diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql index 671d39aae5..670900ea81 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql @@ -164,25 +164,24 @@ BEGIN END; $$; --- call insert_into_ts_kv(); +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; -CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ -DECLARE - insert_size CONSTANT integer := 10000; - insert_counter integer DEFAULT 0; - insert_record RECORD; - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, +CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id, ts_kv_records.key AS key, ts_kv_records.ts AS ts, ts_kv_records.bool_v AS bool_v, ts_kv_records.str_v AS str_v, ts_kv_records.long_v AS long_v, ts_kv_records.dbl_v AS dbl_v - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, + FROM (SELECT entity_id AS entity_id, key_id AS key, ts, bool_v, @@ -190,46 +189,23 @@ DECLARE long_v, dbl_v FROM ts_kv_old - INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records; -BEGIN - OPEN insert_cursor; - LOOP - insert_counter := insert_counter + 1; - FETCH insert_cursor INTO insert_record; - IF NOT FOUND THEN - RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1; - EXIT; - END IF; - INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, - insert_record.long_v, insert_record.dbl_v); - IF MOD(insert_counter, insert_size) = 0 THEN - RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter; - END IF; - END LOOP; - CLOSE insert_cursor; -END; + INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;', path_to_file); + EXECUTE format ('COPY ts_kv FROM %L', path_to_file); +END $$; -- call insert_into_ts_kv_latest(); -CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$ -DECLARE - insert_size CONSTANT integer := 10000; - insert_counter integer DEFAULT 0; - insert_record RECORD; - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, +CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar) LANGUAGE plpgsql AS $$ +BEGIN + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id, ts_kv_latest_records.key AS key, ts_kv_latest_records.ts AS ts, ts_kv_latest_records.bool_v AS bool_v, ts_kv_latest_records.str_v AS str_v, ts_kv_latest_records.long_v AS long_v, ts_kv_latest_records.dbl_v AS dbl_v - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, + FROM (SELECT entity_id AS entity_id, key_id AS key, ts, bool_v, @@ -237,24 +213,8 @@ DECLARE long_v, dbl_v FROM ts_kv_latest_old - INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records; -BEGIN - OPEN insert_cursor; - LOOP - insert_counter := insert_counter + 1; - FETCH insert_cursor INTO insert_record; - IF NOT FOUND THEN - RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1; - EXIT; - END IF; - INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, - insert_record.long_v, insert_record.dbl_v); - IF MOD(insert_counter, insert_size) = 0 THEN - RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter; - END IF; - END LOOP; - CLOSE insert_cursor; + INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;', path_to_file); + EXECUTE format ('COPY ts_kv_latest FROM %L', path_to_file); END; $$; diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql b/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql index 982ec9d85f..30a76aeb4c 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql @@ -96,51 +96,36 @@ BEGIN END; $$; --- call insert_into_ts_kv(); +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; -CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$ +-- call insert_into_ts_kv(); -DECLARE - insert_size CONSTANT integer := 10000; - insert_counter integer DEFAULT 0; - insert_record RECORD; - insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id, - new_ts_kv_records.key AS key, - new_ts_kv_records.ts AS ts, - new_ts_kv_records.bool_v AS bool_v, - new_ts_kv_records.str_v AS str_v, - new_ts_kv_records.long_v AS long_v, - new_ts_kv_records.dbl_v AS dbl_v - FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part, - SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part, - SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part, - SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part, - SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part, - key_id AS key, - ts, - bool_v, - str_v, - long_v, - dbl_v - FROM tenant_ts_kv_old - INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records; +CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$ BEGIN - OPEN insert_cursor; - LOOP - insert_counter := insert_counter + 1; - FETCH insert_cursor INTO insert_record; - IF NOT FOUND THEN - RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1; - EXIT; - END IF; - INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v) - VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v, - insert_record.long_v, insert_record.dbl_v); - IF MOD(insert_counter, insert_size) = 0 THEN - RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter; - END IF; - END LOOP; - CLOSE insert_cursor; + + EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id, + new_ts_kv_records.key AS key, + new_ts_kv_records.ts AS ts, + new_ts_kv_records.bool_v AS bool_v, + new_ts_kv_records.str_v AS str_v, + new_ts_kv_records.long_v AS long_v, + new_ts_kv_records.dbl_v AS dbl_v + FROM (SELECT entity_id AS entity_id, + key_id AS key, + ts, + bool_v, + str_v, + long_v, + dbl_v + FROM tenant_ts_kv_old + INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records) TO %L;', path_to_file); + EXECUTE format ('COPY ts_kv FROM %L', path_to_file); END; $$; diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java index 7afa422460..5f084c7a9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java @@ -34,6 +34,9 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { protected static final String CALL_REGEX = "call "; protected static final String DROP_TABLE = "DROP TABLE "; protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS "; + protected static final String TS_KV_SQL = "ts_kv.sql"; + protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public"; + protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR"; @Value("${spring.datasource.url}") protected String dbUrl; diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index a06ef0fe05..bca71fc539 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -16,12 +16,17 @@ package org.thingsboard.server.service.install; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.SqlTsDao; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; @@ -37,6 +42,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe @Value("${sql.postgres.ts_key_value_partitioning:MONTHS}") private String partitionType; + private static final String TS_KV_LATEST_SQL = "ts_kv_latest.sql"; private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql"; private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql"; private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql"; @@ -49,15 +55,13 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)"; private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; - private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; - private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)"; + private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest(IN path_to_file varchar)"; private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE; private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE; private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; - private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; - private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD; private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD; @@ -94,9 +98,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe } executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); - executeQuery(conn, CALL_INSERT_INTO_TS_KV); - executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); - executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); + + Path pathToTempTsKvFile; + Path pathToTempTsKvLatestFile; + if (SystemUtils.IS_OS_WINDOWS) { + log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR); + Path pathToDir; + String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR"); + if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) { + log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(thingsboardWindowsUpgradeDir); + } else { + log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER); + } + log.info("Directory: {} will be used for creation temporary upgrade files!", pathToDir); + try { + Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql"); + Path tsKvLatestFile = Files.createTempFile(pathToDir, "ts_kv_latest", ".sql"); + pathToTempTsKvFile = tsKvFile.toAbsolutePath(); + pathToTempTsKvLatestFile = tsKvLatestFile.toAbsolutePath(); + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')"); + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');"); + } catch (IOException | SecurityException e) { + throw new RuntimeException("Failed to create time-series upgrade files due to: " + e); + } + } else { + Path tempDirPath = Files.createTempDirectory("ts_kv"); + File tempDirAsFile = tempDirPath.toFile(); + boolean writable = tempDirAsFile.setWritable(true, false); + boolean readable = tempDirAsFile.setReadable(true, false); + boolean executable = tempDirAsFile.setExecutable(true, false); + if (writable && readable && executable) { + pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath(); + pathToTempTsKvLatestFile = tempDirPath.resolve(TS_KV_LATEST_SQL).toAbsolutePath(); + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')"); + executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE); + executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');"); + } else { + throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!"); + } + } + if (pathToTempTsKvFile.toFile().exists() && pathToTempTsKvLatestFile.toFile().exists()) { + boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete(); + if (deleteTsKvFile) { + log.info("Successfully deleted the temp file for ts_kv table upgrade!"); + } + boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete(); + if (deleteTsKvLatestFile) { + log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!"); + } + } executeQuery(conn, DROP_TABLE_TS_KV_OLD); executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD); diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 21bdb500d9..a7f243d7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.install; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; @@ -23,6 +25,9 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; import org.thingsboard.server.dao.util.TimescaleDBTsDao; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Connection; @@ -47,14 +52,13 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()"; private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()"; private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()"; - private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()"; + private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)"; private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()"; private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE; private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE; private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE; private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY; - private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV; private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST; private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE; @@ -63,7 +67,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE; private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE; private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY; - private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; + private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV; private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST; @Autowired @@ -91,7 +95,49 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, CALL_INSERT_INTO_DICTIONARY); - executeQuery(conn, CALL_INSERT_INTO_TS_KV); + + Path pathToTempTsKvFile; + if (SystemUtils.IS_OS_WINDOWS) { + Path pathToDir; + log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR); + String thingsboardWindowsUpgradeDir = System.getenv(THINGSBOARD_WINDOWS_UPGRADE_DIR); + if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) { + log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(thingsboardWindowsUpgradeDir); + } else { + log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR); + pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER); + } + log.info("Directory: {} will be used for creation temporary upgrade file!", pathToDir); + try { + Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql"); + pathToTempTsKvFile = tsKvFile.toAbsolutePath(); + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')"); + pathToTempTsKvFile.toFile().deleteOnExit(); + } catch (IOException | SecurityException e) { + throw new RuntimeException("Failed to create time-series upgrade files due to: " + e); + } + } else { + Path tempDirPath = Files.createTempDirectory("ts_kv"); + File tempDirAsFile = tempDirPath.toFile(); + boolean writable = tempDirAsFile.setWritable(true, false); + boolean readable = tempDirAsFile.setReadable(true, false); + boolean executable = tempDirAsFile.setExecutable(true, false); + if (writable && readable && executable) { + pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath(); + executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')"); + } else { + throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!"); + } + } + + if (pathToTempTsKvFile.toFile().exists()) { + boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete(); + if (deleteTsKvFile) { + log.info("Successfully deleted the temp file for ts_kv table upgrade!"); + } + } + executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST); executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE); @@ -100,7 +146,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY); executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY); - executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV); + executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV); executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST); executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;"); diff --git a/docker/.env b/docker/.env index f943e1a8f2..b632300d5f 100644 --- a/docker/.env +++ b/docker/.env @@ -1,3 +1,4 @@ +TB_QUEUE_TYPE=kafka DOCKER_REPO=thingsboard diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 9ef71cf590..65eeca03e1 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -32,6 +32,32 @@ function additionalComposeArgs() { echo $ADDITIONAL_COMPOSE_ARGS } +function additionalComposeQueueArgs() { + source .env + ADDITIONAL_COMPOSE_QUEUE_ARGS="" + case $TB_QUEUE_TYPE in + kafka) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.kafka.yml" + ;; + aws-sqs) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml" + ;; + pubsub) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.pubsub.yml" + ;; + rabbitmq) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.rabbitmq.yml" + ;; + service-bus) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" + ;; + *) + echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + exit 1 + esac + echo $ADDITIONAL_COMPOSE_QUEUE_ARGS +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml new file mode 100644 index 0000000000..e58917043b --- /dev/null +++ b/docker/docker-compose.aws-sqs.yml @@ -0,0 +1,71 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-js-executor: + env_file: + - queue-aws-sqs.env + tb-core1: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + - redis + tb-core2: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + - redis + tb-rule-engine1: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + - redis + tb-rule-engine2: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + - redis + tb-mqtt-transport1: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + tb-mqtt-transport2: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + tb-http-transport1: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + tb-http-transport2: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper + tb-coap-transport: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper \ No newline at end of file diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 635c7a5dac..20231292ce 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -38,7 +38,7 @@ services: env_file: - tb-node.hybrid.env depends_on: - - kafka + - zookeeper - redis - postgres - cassandra @@ -46,7 +46,7 @@ services: env_file: - tb-node.hybrid.env depends_on: - - kafka + - zookeeper - redis - postgres - cassandra @@ -54,7 +54,7 @@ services: env_file: - tb-node.hybrid.env depends_on: - - kafka + - zookeeper - redis - postgres - cassandra @@ -62,7 +62,7 @@ services: env_file: - tb-node.hybrid.env depends_on: - - kafka + - zookeeper - redis - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml new file mode 100644 index 0000000000..c9b235008b --- /dev/null +++ b/docker/docker-compose.kafka.yml @@ -0,0 +1,82 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + kafka: + restart: always + image: "wurstmeister/kafka:2.12-2.3.0" + ports: + - "9092:9092" + env_file: + - kafka.env + depends_on: + - zookeeper + tb-js-executor: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-core1: + env_file: + - queue-kafka.env + depends_on: + - kafka + - redis + tb-core2: + env_file: + - queue-kafka.env + depends_on: + - kafka + - redis + tb-rule-engine1: + env_file: + - queue-kafka.env + depends_on: + - kafka + - redis + tb-rule-engine2: + env_file: + - queue-kafka.env + depends_on: + - kafka + - redis + tb-mqtt-transport1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-mqtt-transport2: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-http-transport1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-http-transport2: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-coap-transport: + env_file: + - queue-kafka.env + depends_on: + - kafka diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 3d19e5f968..28fe5a55e2 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -31,27 +31,27 @@ services: env_file: - tb-node.postgres.env depends_on: - - kafka + - zookeeper - redis - postgres tb-core2: env_file: - tb-node.postgres.env depends_on: - - kafka + - zookeeper - redis - postgres tb-rule-engine1: env_file: - tb-node.postgres.env depends_on: - - kafka + - zookeeper - redis - postgres tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: - - kafka + - zookeeper - redis - postgres diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml new file mode 100644 index 0000000000..1ec178ee46 --- /dev/null +++ b/docker/docker-compose.pubsub.yml @@ -0,0 +1,71 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-js-executor: + env_file: + - queue-pubsub.env.env + tb-core1: + env_file: + - queue-pubsub.env.env + depends_on: + - zookeeper + - redis + tb-core2: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + - redis + tb-rule-engine1: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + - redis + tb-rule-engine2: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + - redis + tb-mqtt-transport1: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + tb-mqtt-transport2: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + tb-http-transport1: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + tb-http-transport2: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper + tb-coap-transport: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml new file mode 100644 index 0000000000..06662b39b6 --- /dev/null +++ b/docker/docker-compose.rabbitmq.yml @@ -0,0 +1,71 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-js-executor: + env_file: + - queue-rabbitmq.env + tb-core1: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + - redis + tb-core2: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + - redis + tb-rule-engine1: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + - redis + tb-rule-engine2: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + - redis + tb-mqtt-transport1: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + tb-mqtt-transport2: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + tb-http-transport1: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + tb-http-transport2: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper + tb-coap-transport: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper \ No newline at end of file diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml new file mode 100644 index 0000000000..01dd863792 --- /dev/null +++ b/docker/docker-compose.service-bus.yml @@ -0,0 +1,71 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-js-executor: + env_file: + - queue-service-bus.env + tb-core1: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + - redis + tb-core2: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + - redis + tb-rule-engine1: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + - redis + tb-rule-engine2: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + - redis + tb-mqtt-transport1: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + tb-mqtt-transport2: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + tb-http-transport1: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + tb-http-transport2: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper + tb-coap-transport: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9061f3e2de..fbb8ee262d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,15 +26,6 @@ services: environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - kafka: - restart: always - image: "wurstmeister/kafka:2.12-2.3.0" - ports: - - "9092:9092" - env_file: - - kafka.env - depends_on: - - zookeeper redis: restart: always image: redis:4.0 @@ -46,8 +37,6 @@ services: scale: 20 env_file: - tb-js-executor.env - depends_on: - - kafka tb-core1: restart: always image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}" @@ -67,7 +56,7 @@ services: - ./tb-node/conf:/config - ./tb-node/log:/var/log/thingsboard depends_on: - - kafka + - zookeeper - redis - tb-js-executor - tb-rule-engine1 @@ -91,7 +80,7 @@ services: - ./tb-node/conf:/config - ./tb-node/log:/var/log/thingsboard depends_on: - - kafka + - zookeeper - redis - tb-js-executor - tb-rule-engine1 @@ -115,7 +104,7 @@ services: - ./tb-node/conf:/config - ./tb-node/log:/var/log/thingsboard depends_on: - - kafka + - zookeeper - redis - tb-js-executor tb-rule-engine2: @@ -137,7 +126,7 @@ services: - ./tb-node/conf:/config - ./tb-node/log:/var/log/thingsboard depends_on: - - kafka + - zookeeper - redis - tb-js-executor tb-mqtt-transport1: @@ -153,7 +142,7 @@ services: - ./tb-transports/mqtt/conf:/config - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport depends_on: - - kafka + - zookeeper tb-mqtt-transport2: restart: always image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" @@ -167,7 +156,7 @@ services: - ./tb-transports/mqtt/conf:/config - ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport depends_on: - - kafka + - zookeeper tb-http-transport1: restart: always image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" @@ -181,7 +170,7 @@ services: - ./tb-transports/http/conf:/config - ./tb-transports/http/log:/var/log/tb-http-transport depends_on: - - kafka + - zookeeper tb-http-transport2: restart: always image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" @@ -195,7 +184,7 @@ services: - ./tb-transports/http/conf:/config - ./tb-transports/http/log:/var/log/tb-http-transport depends_on: - - kafka + - zookeeper tb-coap-transport: restart: always image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" @@ -209,7 +198,7 @@ services: - ./tb-transports/coap/conf:/config - ./tb-transports/coap/log:/var/log/tb-coap-transport depends_on: - - kafka + - zookeeper tb-web-ui1: restart: always image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}" diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index cabb3c190d..d780bd035b 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -41,14 +41,16 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index e935bc52e6..68f970f6b3 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -19,6 +19,8 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS down -v +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 08041410f5..95bd60e4ff 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -19,6 +19,8 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 492cc12f1a..e7392b38ca 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -19,6 +19,8 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS stop +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 0182fbdcdc..0a43b14c3e 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -19,7 +19,9 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d --no-deps --build $@ +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index e62dbdc438..9d6a7243f6 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -40,12 +40,14 @@ set -e source compose-utils.sh +ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? + ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/queue-aws-sqs.env b/docker/queue-aws-sqs.env new file mode 100644 index 0000000000..1cb9fd65dd --- /dev/null +++ b/docker/queue-aws-sqs.env @@ -0,0 +1,4 @@ +TB_QUEUE_TYPE=aws-sqs +TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY +TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET +TB_QUEUE_AWS_SQS_REGION=YOUR_REGION diff --git a/docker/queue-kafka.env b/docker/queue-kafka.env new file mode 100644 index 0000000000..63107942fb --- /dev/null +++ b/docker/queue-kafka.env @@ -0,0 +1,2 @@ +TB_QUEUE_TYPE=kafka +TB_KAFKA_SERVERS=kafka:9092 diff --git a/docker/queue-pubsub.env b/docker/queue-pubsub.env new file mode 100644 index 0000000000..3bc62a1560 --- /dev/null +++ b/docker/queue-pubsub.env @@ -0,0 +1,3 @@ +TB_QUEUE_TYPE=pubsub +TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID +TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT \ No newline at end of file diff --git a/docker/queue-rabbitmq.env b/docker/queue-rabbitmq.env new file mode 100644 index 0000000000..7c355a4910 --- /dev/null +++ b/docker/queue-rabbitmq.env @@ -0,0 +1,5 @@ +TB_QUEUE_TYPE=rabbitmq +TB_QUEUE_RABBIT_MQ_HOST=localhost +TB_QUEUE_RABBIT_MQ_PORT=5672 +TB_QUEUE_RABBIT_MQ_USERNAME=YOUR_USERNAME +TB_QUEUE_RABBIT_MQ_PASSWORD=YOUR_PASSWORD \ No newline at end of file diff --git a/docker/queue-service-bus.env b/docker/queue-service-bus.env new file mode 100644 index 0000000000..07bd0dc061 --- /dev/null +++ b/docker/queue-service-bus.env @@ -0,0 +1,4 @@ +TB_QUEUE_TYPE=service-bus +TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME=YOUR_NAMESPACE_NAME +TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME=YOUR_SAS_KEY_NAME +TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY \ No newline at end of file diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 8a75904185..406367a192 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 - -TB_QUEUE_TYPE=kafka -TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index e62424290f..41fb76453d 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 - -TB_QUEUE_TYPE=kafka -TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index b66073ea44..e080906549 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,6 +1,4 @@ -TB_QUEUE_TYPE=kafka REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests -TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info LOG_FOLDER=logs LOGGER_FILENAME=tb-js-executor-%DATE%.log diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 54c35f355c..04f6ed8e98 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 - -TB_QUEUE_TYPE=kafka -TB_KAFKA_SERVERS=kafka:9092 \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index 12cdc7d035..bc0a3f5ee5 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -2,8 +2,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 -TB_QUEUE_TYPE=kafka -TB_KAFKA_SERVERS=kafka:9092 JS_EVALUATOR=remote TRANSPORT_TYPE=remote CACHE_TYPE=redis diff --git a/k8s/README.md b/k8s/README.md index 2a62329219..6e888352dc 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -40,6 +40,26 @@ Where: ## Running +Execute the following command to deploy thirdparty resources: + +` +$ ./k8s-deploy-thirdparty.sh +` + +Get list of the running tb-redis pods and verify that all of them are in running state: + +` +$ kubectl get pods -l app=tb-redis +` + +Execute the following command to create redis cluster: + +` +$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ') +` + +Type **'yes'** when prompted. + Execute the following command to deploy resources: ` diff --git a/k8s/k8s-delete-resources.sh b/k8s/k8s-delete-resources.sh index 27be2b5074..21efc56bc7 100755 --- a/k8s/k8s-delete-resources.sh +++ b/k8s/k8s-delete-resources.sh @@ -19,3 +19,4 @@ set -e kubectl config set-context $(kubectl config current-context) --namespace=thingsboard kubectl delete -f thingsboard.yml +kubectl delete -f thirdparty.yml diff --git a/k8s/k8s-delete-thirdparty.sh b/k8s/k8s-delete-thirdparty.sh new file mode 100755 index 0000000000..a93b2b809e --- /dev/null +++ b/k8s/k8s-delete-thirdparty.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard +kubectl delete -f thirdparty.yml diff --git a/k8s/k8s-deploy-thirdparty.sh b/k8s/k8s-deploy-thirdparty.sh new file mode 100755 index 0000000000..adc526506a --- /dev/null +++ b/k8s/k8s-deploy-thirdparty.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +kubectl apply -f tb-namespace.yml +kubectl config set-context $(kubectl config current-context) --namespace=thingsboard +kubectl apply -f thirdparty.yml diff --git a/k8s/thingsboard.yml b/k8s/thingsboard.yml index 8f50f34f8e..d1f598b6d3 100644 --- a/k8s/thingsboard.yml +++ b/k8s/thingsboard.yml @@ -14,169 +14,6 @@ # limitations under the License. # -apiVersion: apps/v1 -kind: Deployment -metadata: - name: zookeeper - namespace: thingsboard -spec: - selector: - matchLabels: - app: zookeeper - template: - metadata: - labels: - app: zookeeper - spec: - containers: - - name: server - imagePullPolicy: Always - image: zookeeper:3.5 - ports: - - containerPort: 2181 - readinessProbe: - periodSeconds: 5 - tcpSocket: - port: 2181 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 2181 - env: - - name: ZOO_MY_ID - value: "1" - - name: ZOO_SERVERS - value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181" - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: zookeeper - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: zookeeper - ports: - - name: zk-port - port: 2181 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tb-kafka - namespace: thingsboard -spec: - selector: - matchLabels: - app: tb-kafka - template: - metadata: - labels: - app: tb-kafka - spec: - containers: - - name: server - imagePullPolicy: Always - image: wurstmeister/kafka:2.12-2.2.1 - ports: - - containerPort: 9092 - readinessProbe: - periodSeconds: 20 - tcpSocket: - port: 9092 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 9092 - env: - - name: KAFKA_ZOOKEEPER_CONNECT - value: "zookeeper:2181" - - name: KAFKA_LISTENERS - value: "INSIDE://:9093,OUTSIDE://:9092" - - name: KAFKA_ADVERTISED_LISTENERS - value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092" - - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP - value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" - - name: KAFKA_INTER_BROKER_LISTENER_NAME - value: "INSIDE" - - name: KAFKA_CREATE_TOPICS - value: "js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" - - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE - value: "false" - - name: KAFKA_LOG_RETENTION_BYTES - value: "1073741824" - - name: KAFKA_LOG_SEGMENT_BYTES - value: "268435456" - - name: KAFKA_LOG_RETENTION_MS - value: "300000" - - name: KAFKA_LOG_CLEANUP_POLICY - value: "delete" - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: tb-kafka - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: tb-kafka - ports: - - name: tb-kafka-port - port: 9092 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tb-redis - namespace: thingsboard -spec: - selector: - matchLabels: - app: tb-redis - template: - metadata: - labels: - app: tb-redis - spec: - containers: - - name: server - imagePullPolicy: Always - image: redis:4.0 - ports: - - containerPort: 6379 - readinessProbe: - periodSeconds: 5 - tcpSocket: - port: 6379 - livenessProbe: - periodSeconds: 5 - tcpSocket: - port: 6379 - volumeMounts: - - mountPath: /data - name: redis-data - volumes: - - name: redis-data - emptyDir: {} - restartPolicy: Always ---- -apiVersion: v1 -kind: Service -metadata: - name: tb-redis - namespace: thingsboard -spec: - type: ClusterIP - selector: - app: tb-redis - ports: - - name: tb-redis-port - port: 6379 ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -267,6 +104,10 @@ spec: value: "redis" - name: REDIS_HOST value: "tb-redis" + - name: REDIS_CONNECTION_TYPE + value: "cluster" + - name: REDIS_NODES + value: "tb-redis:6379" - name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE value: "false" envFrom: diff --git a/k8s/thirdparty.yml b/k8s/thirdparty.yml new file mode 100644 index 0000000000..2638bf0af5 --- /dev/null +++ b/k8s/thirdparty.yml @@ -0,0 +1,301 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zookeeper + namespace: thingsboard +spec: + serviceName: "zookeeper" + replicas: 3 + podManagementPolicy: Parallel + selector: + matchLabels: + app: zookeeper + template: + metadata: + labels: + app: zookeeper + spec: + containers: + - name: zookeeper + imagePullPolicy: Always + image: zookeeper:3.5 + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: election + readinessProbe: + periodSeconds: 60 + tcpSocket: + port: 2181 + livenessProbe: + periodSeconds: 60 + tcpSocket: + port: 2181 + env: + - name: ZOO_SERVERS + value: "server.0=zookeeper-0.zookeeper:2888:3888;2181 server.1=zookeeper-1.zookeeper:2888:3888;2181 server.2=zookeeper-2.zookeeper:2888:3888;2181" + - name: JVMFLAGS + value: "-Dzookeeper.electionPortBindRetry=0" + volumeMounts: + - name: data + mountPath: /data + readOnly: false + initContainers: + - command: + - /bin/bash + - -c + - |- + set -ex; + mkdir -p "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR"; + chown "$ZOO_USER:$ZOO_USER" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR" + if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then + echo $HOSTNAME| rev | cut -d "-" -f1 | rev > "$ZOO_DATA_DIR/myid" + fi + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: zookeeper:3.5 + imagePullPolicy: IfNotPresent + name: zookeeper-init + securityContext: + runAsUser: 0 + volumeMounts: + - name: data + mountPath: /data + readOnly: false + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 2181 + targetPort: 2181 + name: client + - port: 2888 + targetPort: 2888 + name: server + - port: 3888 + targetPort: 3888 + name: election + selector: + app: zookeeper +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tb-kafka + namespace: thingsboard +spec: + serviceName: "tb-kafka" + replicas: 3 + podManagementPolicy: Parallel + selector: + matchLabels: + app: tb-kafka + template: + metadata: + labels: + app: tb-kafka + spec: + containers: + - name: tb-kafka + imagePullPolicy: Always + image: wurstmeister/kafka:2.12-2.2.1 + ports: + - containerPort: 9092 + name: kafka-int + readinessProbe: + periodSeconds: 5 + timeoutSeconds: 5 + tcpSocket: + port: 9092 + initialDelaySeconds: 60 + livenessProbe: + timeoutSeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 9092 + initialDelaySeconds: 80 + env: + - name: BROKER_ID_COMMAND + value: "hostname | cut -d'-' -f3" + - name: KAFKA_ZOOKEEPER_CONNECT + value: "zookeeper:2181" + - name: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS + value: "60000" + - name: KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE + value: "true" + - name: KAFKA_LISTENERS + value: "INSIDE://:9092" + - name: KAFKA_ADVERTISED_LISTENERS + value: "INSIDE://:9092" + - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP + value: "INSIDE:PLAINTEXT" + - name: KAFKA_INTER_BROKER_LISTENER_NAME + value: "INSIDE" + - name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE + value: "true" + - name: KAFKA_CREATE_TOPICS + value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600" + - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE + value: "false" + - name: KAFKA_LOG_RETENTION_BYTES + value: "1073741824" + - name: KAFKA_LOG_SEGMENT_BYTES + value: "268435456" + - name: KAFKA_LOG_RETENTION_MS + value: "300000" + - name: KAFKA_LOG_CLEANUP_POLICY + value: "delete" + - name: KAFKA_PORT + value: "9092" + - name: KAFKA_LOG_DIRS + value: "/kafka-logs" + volumeMounts: + - name: logs + mountPath: /kafka-logs + subPath: logs + volumeClaimTemplates: + - metadata: + name: logs + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-kafka + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 9092 + targetPort: 9092 + name: kafka-int + selector: + app: tb-kafka +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: tb-redis + namespace: thingsboard +data: + update-node.sh: | + #!/bin/sh + REDIS_NODES="/data/nodes.conf" + sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES} + exec "$@" + redis.conf: |+ + cluster-enabled yes + cluster-require-full-coverage no + cluster-node-timeout 15000 + cluster-config-file /data/nodes.conf + cluster-migration-barrier 1 + appendonly yes + protected-mode no +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tb-redis + namespace: thingsboard +spec: + serviceName: server + replicas: 6 + selector: + matchLabels: + app: tb-redis + template: + metadata: + labels: + app: tb-redis + spec: + containers: + - name: redis + image: redis:5.0.1-alpine + ports: + - containerPort: 6379 + name: client + - containerPort: 16379 + name: gossip + command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: conf + mountPath: /conf + readOnly: false + - name: data + mountPath: /data + readOnly: false + volumes: + - name: conf + configMap: + name: tb-redis + defaultMode: 0755 + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: tb-redis + namespace: thingsboard +spec: + type: ClusterIP + ports: + - port: 6379 + targetPort: 6379 + name: client + - port: 16379 + targetPort: 16379 + name: gossip + selector: + app: tb-redis