tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge branch 'master' into develop/3.0

pull/2763/head
Igor Kulikov 6 years ago
parent
2d235cf9eb b9bc280d57
commit
6bab8bd6d3
37 changed files with 980 additions and 328 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 80
      application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
  2. 69
      application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
  3. 3
      application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java
  4. 67
      application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
  5. 56
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  6. 1
      docker/.env
  7. 26
      docker/compose-utils.sh
  8. 71
      docker/docker-compose.aws-sqs.yml
  9. 8
      docker/docker-compose.hybrid.yml
  10. 82
      docker/docker-compose.kafka.yml
  11. 8
      docker/docker-compose.postgres.yml
  12. 71
      docker/docker-compose.pubsub.yml
  13. 71
      docker/docker-compose.rabbitmq.yml
  14. 71
      docker/docker-compose.service-bus.yml
  15. 29
      docker/docker-compose.yml
  16. 6
      docker/docker-install-tb.sh
  17. 4
      docker/docker-remove-services.sh
  18. 4
      docker/docker-start-services.sh
  19. 4
      docker/docker-stop-services.sh
  20. 6
      docker/docker-update-service.sh
  21. 8
      docker/docker-upgrade-tb.sh
  22. 4
      docker/queue-aws-sqs.env
  23. 2
      docker/queue-kafka.env
  24. 3
      docker/queue-pubsub.env
  25. 5
      docker/queue-rabbitmq.env
  26. 4
      docker/queue-service-bus.env
  27. 3
      docker/tb-coap-transport.env
  28. 3
      docker/tb-http-transport.env
  29. 2
      docker/tb-js-executor.env
  30. 3
      docker/tb-mqtt-transport.env
  31. 2
      docker/tb-node.env
  32. 20
      k8s/README.md
  33. 1
      k8s/k8s-delete-resources.sh
  34. 21
      k8s/k8s-delete-thirdparty.sh
  35. 22
      k8s/k8s-deploy-thirdparty.sh
  36. 167
      k8s/thingsboard.yml
  37. 301
      k8s/thirdparty.yml

80
application/src/main/data/upgrade/2.4.3/schema_update_psql_ts.sql
View File

@ -164,25 +164,24 @@ BEGIN
END;
$$;
-- call insert_into_ts_kv();
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
BEGIN
EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
ts_kv_records.key AS key,
ts_kv_records.ts AS ts,
ts_kv_records.bool_v AS bool_v,
ts_kv_records.str_v AS str_v,
ts_kv_records.long_v AS long_v,
ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
@ -190,46 +189,23 @@ DECLARE
long_v,
dbl_v
FROM ts_kv_old
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the partitioned ts_kv!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
END;
INNER JOIN ts_kv_dictionary ON (ts_kv_old.key = ts_kv_dictionary.key)) AS ts_kv_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
END
$$;
-- call insert_into_ts_kv_latest();
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest() LANGUAGE plpgsql AS $$
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
CREATE OR REPLACE PROCEDURE insert_into_ts_kv_latest(IN path_to_file varchar) LANGUAGE plpgsql AS $$
BEGIN
EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
ts_kv_latest_records.key AS key,
ts_kv_latest_records.ts AS ts,
ts_kv_latest_records.bool_v AS bool_v,
ts_kv_latest_records.str_v AS str_v,
ts_kv_latest_records.long_v AS long_v,
ts_kv_latest_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
@ -237,24 +213,8 @@ DECLARE
long_v,
dbl_v
FROM ts_kv_latest_old
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records;
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv_latest(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the ts_kv_latest!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
INNER JOIN ts_kv_dictionary ON (ts_kv_latest_old.key = ts_kv_dictionary.key)) AS ts_kv_latest_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv_latest FROM %L', path_to_file);
END;
$$;

69
application/src/main/data/upgrade/2.4.3/schema_update_timescale_ts.sql
View File

@ -96,51 +96,36 @@ BEGIN
END;
$$;
-- call insert_into_ts_kv();
CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS
$$
BEGIN
uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) ||
'-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12);
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE PROCEDURE insert_into_ts_kv() LANGUAGE plpgsql AS $$
-- call insert_into_ts_kv();
DECLARE
insert_size CONSTANT integer := 10000;
insert_counter integer DEFAULT 0;
insert_record RECORD;
insert_cursor CURSOR FOR SELECT CONCAT(entity_id_uuid_first_part, '-', entity_id_uuid_second_part, '-1', entity_id_uuid_third_part, '-', entity_id_uuid_fourth_part, '-', entity_id_uuid_fifth_part)::uuid AS entity_id,
new_ts_kv_records.key AS key,
new_ts_kv_records.ts AS ts,
new_ts_kv_records.bool_v AS bool_v,
new_ts_kv_records.str_v AS str_v,
new_ts_kv_records.long_v AS long_v,
new_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT SUBSTRING(entity_id, 8, 8) AS entity_id_uuid_first_part,
SUBSTRING(entity_id, 4, 4) AS entity_id_uuid_second_part,
SUBSTRING(entity_id, 1, 3) AS entity_id_uuid_third_part,
SUBSTRING(entity_id, 16, 4) AS entity_id_uuid_fourth_part,
SUBSTRING(entity_id, 20) AS entity_id_uuid_fifth_part,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records;
CREATE OR REPLACE PROCEDURE insert_into_ts_kv(IN path_to_file varchar) LANGUAGE plpgsql AS $$
BEGIN
OPEN insert_cursor;
LOOP
insert_counter := insert_counter + 1;
FETCH insert_cursor INTO insert_record;
IF NOT FOUND THEN
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter - 1;
EXIT;
END IF;
INSERT INTO ts_kv(entity_id, key, ts, bool_v, str_v, long_v, dbl_v)
VALUES (insert_record.entity_id, insert_record.key, insert_record.ts, insert_record.bool_v, insert_record.str_v,
insert_record.long_v, insert_record.dbl_v);
IF MOD(insert_counter, insert_size) = 0 THEN
RAISE NOTICE '% records have been inserted into the new ts_kv table!',insert_counter;
END IF;
END LOOP;
CLOSE insert_cursor;
EXECUTE format ('COPY (SELECT to_uuid(entity_id) AS entity_id,
new_ts_kv_records.key AS key,
new_ts_kv_records.ts AS ts,
new_ts_kv_records.bool_v AS bool_v,
new_ts_kv_records.str_v AS str_v,
new_ts_kv_records.long_v AS long_v,
new_ts_kv_records.dbl_v AS dbl_v
FROM (SELECT entity_id AS entity_id,
key_id AS key,
ts,
bool_v,
str_v,
long_v,
dbl_v
FROM tenant_ts_kv_old
INNER JOIN ts_kv_dictionary ON (tenant_ts_kv_old.key = ts_kv_dictionary.key)) AS new_ts_kv_records) TO %L;', path_to_file);
EXECUTE format ('COPY ts_kv FROM %L', path_to_file);
END;
$$;

3
application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java
View File

@ -34,6 +34,9 @@ public abstract class AbstractSqlTsDatabaseUpgradeService {
protected static final String CALL_REGEX = "call ";
protected static final String DROP_TABLE = "DROP TABLE ";
protected static final String DROP_PROCEDURE_IF_EXISTS = "DROP PROCEDURE IF EXISTS ";
protected static final String TS_KV_SQL = "ts_kv.sql";
protected static final String PATH_TO_USERS_PUBLIC_FOLDER = "C:\\Users\\Public";
protected static final String THINGSBOARD_WINDOWS_UPGRADE_DIR = "THINGSBOARD_WINDOWS_UPGRADE_DIR";
@Value("${spring.datasource.url}")
protected String dbUrl;

67
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java
View File

@ -16,12 +16,17 @@
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlTsDao;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
@ -37,6 +42,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
@Value("${sql.postgres.ts_key_value_partitioning:MONTHS}")
private String partitionType;
private static final String TS_KV_LATEST_SQL = "ts_kv_latest.sql";
private static final String LOAD_FUNCTIONS_SQL = "schema_update_psql_ts.sql";
private static final String LOAD_TTL_FUNCTIONS_SQL = "schema_update_ttl.sql";
private static final String LOAD_DROP_PARTITIONS_FUNCTIONS_SQL = "schema_update_psql_drop_partitions.sql";
@ -49,15 +55,13 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
private static final String CREATE_PARTITIONS = "create_partitions(IN partition_type varchar)";
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest(IN path_to_file varchar)";
private static final String CALL_CREATE_PARTITION_TS_KV_TABLE = CALL_REGEX + CREATE_PARTITION_TS_KV_TABLE;
private static final String CALL_CREATE_NEW_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_TABLE_TS_KV_OLD = DROP_TABLE + TS_KV_OLD;
private static final String DROP_TABLE_TS_KV_LATEST_OLD = DROP_TABLE + TS_KV_LATEST_OLD;
@ -94,9 +98,58 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe
}
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
Path pathToTempTsKvFile;
Path pathToTempTsKvLatestFile;
if (SystemUtils.IS_OS_WINDOWS) {
log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
Path pathToDir;
String thingsboardWindowsUpgradeDir = System.getenv("THINGSBOARD_WINDOWS_UPGRADE_DIR");
if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
} else {
log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
}
log.info("Directory: {} will be used for creation temporary upgrade files!", pathToDir);
try {
Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
Path tsKvLatestFile = Files.createTempFile(pathToDir, "ts_kv_latest", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
pathToTempTsKvLatestFile = tsKvLatestFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
pathToTempTsKvLatestFile = tempDirPath.resolve(TS_KV_LATEST_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
executeQuery(conn, CALL_CREATE_NEW_TS_KV_LATEST_TABLE);
executeQuery(conn, "call insert_into_ts_kv_latest('" + pathToTempTsKvLatestFile + "');");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists() && pathToTempTsKvLatestFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
boolean deleteTsKvLatestFile = pathToTempTsKvLatestFile.toFile().delete();
if (deleteTsKvLatestFile) {
log.info("Successfully deleted the temp file for ts_kv_latest table upgrade!");
}
}
executeQuery(conn, DROP_TABLE_TS_KV_OLD);
executeQuery(conn, DROP_TABLE_TS_KV_LATEST_OLD);

56
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
View File

@ -16,6 +16,8 @@
package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
@ -23,6 +25,9 @@ import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
@ -47,14 +52,13 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String CREATE_NEW_TS_KV_TABLE = "create_new_ts_kv_table()";
private static final String CREATE_TS_KV_DICTIONARY_TABLE = "create_ts_kv_dictionary_table()";
private static final String INSERT_INTO_DICTIONARY = "insert_into_dictionary()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv()";
private static final String INSERT_INTO_TS_KV = "insert_into_ts_kv(IN path_to_file varchar)";
private static final String INSERT_INTO_TS_KV_LATEST = "insert_into_ts_kv_latest()";
private static final String CALL_CREATE_TS_KV_LATEST_TABLE = CALL_REGEX + CREATE_TS_KV_LATEST_TABLE;
private static final String CALL_CREATE_NEW_TENANT_TS_KV_TABLE = CALL_REGEX + CREATE_NEW_TS_KV_TABLE;
private static final String CALL_CREATE_TS_KV_DICTIONARY_TABLE = CALL_REGEX + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String CALL_INSERT_INTO_DICTIONARY = CALL_REGEX + INSERT_INTO_DICTIONARY;
private static final String CALL_INSERT_INTO_TS_KV = CALL_REGEX + INSERT_INTO_TS_KV;
private static final String CALL_INSERT_INTO_TS_KV_LATEST = CALL_REGEX + INSERT_INTO_TS_KV_LATEST;
private static final String DROP_OLD_TENANT_TS_KV_TABLE = DROP_TABLE + TENANT_TS_KV_OLD_TABLE;
@ -63,7 +67,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
private static final String DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY = DROP_PROCEDURE_IF_EXISTS + CREATE_NEW_TS_KV_TABLE;
private static final String DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE = DROP_PROCEDURE_IF_EXISTS + CREATE_TS_KV_DICTIONARY_TABLE;
private static final String DROP_PROCEDURE_INSERT_INTO_DICTIONARY = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_DICTIONARY;
private static final String DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV;
private static final String DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST = DROP_PROCEDURE_IF_EXISTS + INSERT_INTO_TS_KV_LATEST;
@Autowired
@ -91,7 +95,49 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, CALL_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, CALL_INSERT_INTO_DICTIONARY);
executeQuery(conn, CALL_INSERT_INTO_TS_KV);
Path pathToTempTsKvFile;
if (SystemUtils.IS_OS_WINDOWS) {
Path pathToDir;
log.info("Lookup for environment variable: {} ...", THINGSBOARD_WINDOWS_UPGRADE_DIR);
String thingsboardWindowsUpgradeDir = System.getenv(THINGSBOARD_WINDOWS_UPGRADE_DIR);
if (StringUtils.isNotEmpty(thingsboardWindowsUpgradeDir)) {
log.info("Environment variable: {} was found!", THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(thingsboardWindowsUpgradeDir);
} else {
log.info("Failed to lookup environment variable: {}", THINGSBOARD_WINDOWS_UPGRADE_DIR);
pathToDir = Paths.get(PATH_TO_USERS_PUBLIC_FOLDER);
}
log.info("Directory: {} will be used for creation temporary upgrade file!", pathToDir);
try {
Path tsKvFile = Files.createTempFile(pathToDir, "ts_kv", ".sql");
pathToTempTsKvFile = tsKvFile.toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
pathToTempTsKvFile.toFile().deleteOnExit();
} catch (IOException | SecurityException e) {
throw new RuntimeException("Failed to create time-series upgrade files due to: " + e);
}
} else {
Path tempDirPath = Files.createTempDirectory("ts_kv");
File tempDirAsFile = tempDirPath.toFile();
boolean writable = tempDirAsFile.setWritable(true, false);
boolean readable = tempDirAsFile.setReadable(true, false);
boolean executable = tempDirAsFile.setExecutable(true, false);
if (writable && readable && executable) {
pathToTempTsKvFile = tempDirPath.resolve(TS_KV_SQL).toAbsolutePath();
executeQuery(conn, "call insert_into_ts_kv('" + pathToTempTsKvFile + "')");
} else {
throw new RuntimeException("Failed to grant write permissions for the: " + tempDirPath + "folder!");
}
}
if (pathToTempTsKvFile.toFile().exists()) {
boolean deleteTsKvFile = pathToTempTsKvFile.toFile().delete();
if (deleteTsKvFile) {
log.info("Successfully deleted the temp file for ts_kv table upgrade!");
}
}
executeQuery(conn, CALL_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, DROP_OLD_TENANT_TS_KV_TABLE);
@ -100,7 +146,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr
executeQuery(conn, DROP_PROCEDURE_CREATE_TENANT_TS_KV_TABLE_COPY);
executeQuery(conn, DROP_PROCEDURE_CREATE_TS_KV_DICTIONARY_TABLE);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_DICTIONARY);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TENANT_TS_KV);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV);
executeQuery(conn, DROP_PROCEDURE_INSERT_INTO_TS_KV_LATEST);
executeQuery(conn, "ALTER TABLE ts_kv ADD COLUMN IF NOT EXISTS json_v json;");

1
docker/.env
View File

@ -1,3 +1,4 @@
TB_QUEUE_TYPE=kafka
DOCKER_REPO=thingsboard

26
docker/compose-utils.sh
View File

@ -32,6 +32,32 @@ function additionalComposeArgs() {
echo $ADDITIONAL_COMPOSE_ARGS
}
function additionalComposeQueueArgs() {
source .env
ADDITIONAL_COMPOSE_QUEUE_ARGS=""
case $TB_QUEUE_TYPE in
kafka)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.kafka.yml"
;;
aws-sqs)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml"
;;
pubsub)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.pubsub.yml"
;;
rabbitmq)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.rabbitmq.yml"
;;
service-bus)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml"
;;
*)
echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or aws-sqs or pubsub or rabbitmq or service-bus." >&2
exit 1
esac
echo $ADDITIONAL_COMPOSE_QUEUE_ARGS
}
function additionalStartupServices() {
source .env
ADDITIONAL_STARTUP_SERVICES=""

71
docker/docker-compose.aws-sqs.yml
View File

@ -0,0 +1,71 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-js-executor:
env_file:
- queue-aws-sqs.env
tb-core1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-core2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-mqtt-transport2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-coap-transport:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper

8
docker/docker-compose.hybrid.yml
View File

@ -38,7 +38,7 @@ services:
env_file:
- tb-node.hybrid.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
- cassandra
@ -46,7 +46,7 @@ services:
env_file:
- tb-node.hybrid.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
- cassandra
@ -54,7 +54,7 @@ services:
env_file:
- tb-node.hybrid.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
- cassandra
@ -62,7 +62,7 @@ services:
env_file:
- tb-node.hybrid.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
- cassandra

82
docker/docker-compose.kafka.yml
View File

@ -0,0 +1,82 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
kafka:
restart: always
image: "wurstmeister/kafka:2.12-2.3.0"
ports:
- "9092:9092"
env_file:
- kafka.env
depends_on:
- zookeeper
tb-js-executor:
env_file:
- queue-kafka.env
depends_on:
- kafka
tb-core1:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-core2:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-rule-engine1:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-rule-engine2:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-mqtt-transport1:
env_file:
- queue-kafka.env
depends_on:
- kafka
tb-mqtt-transport2:
env_file:
- queue-kafka.env
depends_on:
- kafka
tb-http-transport1:
env_file:
- queue-kafka.env
depends_on:
- kafka
tb-http-transport2:
env_file:
- queue-kafka.env
depends_on:
- kafka
tb-coap-transport:
env_file:
- queue-kafka.env
depends_on:
- kafka

8
docker/docker-compose.postgres.yml
View File

@ -31,27 +31,27 @@ services:
env_file:
- tb-node.postgres.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
tb-core2:
env_file:
- tb-node.postgres.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
tb-rule-engine1:
env_file:
- tb-node.postgres.env
depends_on:
- kafka
- zookeeper
- redis
- postgres
tb-rule-engine2:
env_file:
- tb-node.postgres.env
depends_on:
- kafka
- zookeeper
- redis
- postgres

71
docker/docker-compose.pubsub.yml
View File

@ -0,0 +1,71 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-js-executor:
env_file:
- queue-pubsub.env.env
tb-core1:
env_file:
- queue-pubsub.env.env
depends_on:
- zookeeper
- redis
tb-core2:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
- redis
tb-rule-engine1:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
- redis
tb-rule-engine2:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
tb-mqtt-transport2:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
tb-http-transport1:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
tb-http-transport2:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper
tb-coap-transport:
env_file:
- queue-pubsub.env
depends_on:
- zookeeper

71
docker/docker-compose.rabbitmq.yml
View File

@ -0,0 +1,71 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-js-executor:
env_file:
- queue-rabbitmq.env
tb-core1:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
- redis
tb-core2:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
- redis
tb-rule-engine1:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
- redis
tb-rule-engine2:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
tb-mqtt-transport2:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
tb-http-transport1:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
tb-http-transport2:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper
tb-coap-transport:
env_file:
- queue-rabbitmq.env
depends_on:
- zookeeper

71
docker/docker-compose.service-bus.yml
View File

@ -0,0 +1,71 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-js-executor:
env_file:
- queue-service-bus.env
tb-core1:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
- redis
tb-core2:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
- redis
tb-rule-engine1:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
- redis
tb-rule-engine2:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
tb-mqtt-transport2:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
tb-http-transport1:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
tb-http-transport2:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper
tb-coap-transport:
env_file:
- queue-service-bus.env
depends_on:
- zookeeper

29
docker/docker-compose.yml
View File

@ -26,15 +26,6 @@ services:
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181
kafka:
restart: always
image: "wurstmeister/kafka:2.12-2.3.0"
ports:
- "9092:9092"
env_file:
- kafka.env
depends_on:
- zookeeper
redis:
restart: always
image: redis:4.0
@ -46,8 +37,6 @@ services:
scale: 20
env_file:
- tb-js-executor.env
depends_on:
- kafka
tb-core1:
restart: always
image: "${DOCKER_REPO}/${TB_NODE_DOCKER_NAME}:${TB_VERSION}"
@ -67,7 +56,7 @@ services:
- ./tb-node/conf:/config
- ./tb-node/log:/var/log/thingsboard
depends_on:
- kafka
- zookeeper
- redis
- tb-js-executor
- tb-rule-engine1
@ -91,7 +80,7 @@ services:
- ./tb-node/conf:/config
- ./tb-node/log:/var/log/thingsboard
depends_on:
- kafka
- zookeeper
- redis
- tb-js-executor
- tb-rule-engine1
@ -115,7 +104,7 @@ services:
- ./tb-node/conf:/config
- ./tb-node/log:/var/log/thingsboard
depends_on:
- kafka
- zookeeper
- redis
- tb-js-executor
tb-rule-engine2:
@ -137,7 +126,7 @@ services:
- ./tb-node/conf:/config
- ./tb-node/log:/var/log/thingsboard
depends_on:
- kafka
- zookeeper
- redis
- tb-js-executor
tb-mqtt-transport1:
@ -153,7 +142,7 @@ services:
- ./tb-transports/mqtt/conf:/config
- ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
depends_on:
- kafka
- zookeeper
tb-mqtt-transport2:
restart: always
image: "${DOCKER_REPO}/${MQTT_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@ -167,7 +156,7 @@ services:
- ./tb-transports/mqtt/conf:/config
- ./tb-transports/mqtt/log:/var/log/tb-mqtt-transport
depends_on:
- kafka
- zookeeper
tb-http-transport1:
restart: always
image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@ -181,7 +170,7 @@ services:
- ./tb-transports/http/conf:/config
- ./tb-transports/http/log:/var/log/tb-http-transport
depends_on:
- kafka
- zookeeper
tb-http-transport2:
restart: always
image: "${DOCKER_REPO}/${HTTP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@ -195,7 +184,7 @@ services:
- ./tb-transports/http/conf:/config
- ./tb-transports/http/log:/var/log/tb-http-transport
depends_on:
- kafka
- zookeeper
tb-coap-transport:
restart: always
image: "${DOCKER_REPO}/${COAP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}"
@ -209,7 +198,7 @@ services:
- ./tb-transports/coap/conf:/config
- ./tb-transports/coap/log:/var/log/tb-coap-transport
depends_on:
- kafka
- zookeeper
tb-web-ui1:
restart: always
image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}"

6
docker/docker-install-tb.sh
View File

@ -41,14 +41,16 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
fi
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1

4
docker/docker-remove-services.sh
View File

@ -19,6 +19,8 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS down -v
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS down -v

4
docker/docker-start-services.sh
View File

@ -19,6 +19,8 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d

4
docker/docker-stop-services.sh
View File

@ -19,6 +19,8 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS stop
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS stop

6
docker/docker-update-service.sh
View File

@ -19,7 +19,9 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull $@
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d --no-deps --build $@
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@

8
docker/docker-upgrade-tb.sh
View File

@ -40,12 +40,14 @@ set -e
source compose-utils.sh
ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS pull tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1

4
docker/queue-aws-sqs.env
View File

@ -0,0 +1,4 @@
TB_QUEUE_TYPE=aws-sqs
TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY
TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET
TB_QUEUE_AWS_SQS_REGION=YOUR_REGION

2
docker/queue-kafka.env
View File

@ -0,0 +1,2 @@
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=kafka:9092

3
docker/queue-pubsub.env
View File

@ -0,0 +1,3 @@
TB_QUEUE_TYPE=pubsub
TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID
TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT

5
docker/queue-rabbitmq.env
View File

@ -0,0 +1,5 @@
TB_QUEUE_TYPE=rabbitmq
TB_QUEUE_RABBIT_MQ_HOST=localhost
TB_QUEUE_RABBIT_MQ_PORT=5672
TB_QUEUE_RABBIT_MQ_USERNAME=YOUR_USERNAME
TB_QUEUE_RABBIT_MQ_PASSWORD=YOUR_PASSWORD

4
docker/queue-service-bus.env
View File

@ -0,0 +1,4 @@
TB_QUEUE_TYPE=service-bus
TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME=YOUR_NAMESPACE_NAME
TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME=YOUR_SAS_KEY_NAME
TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY

3
docker/tb-coap-transport.env
View File

@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
COAP_BIND_ADDRESS=0.0.0.0
COAP_BIND_PORT=5683
COAP_TIMEOUT=10000
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=kafka:9092

3
docker/tb-http-transport.env
View File

@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
HTTP_BIND_ADDRESS=0.0.0.0
HTTP_BIND_PORT=8081
HTTP_REQUEST_TIMEOUT=60000
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=kafka:9092

2
docker/tb-js-executor.env
View File

@ -1,6 +1,4 @@
TB_QUEUE_TYPE=kafka
REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests
TB_KAFKA_SERVERS=kafka:9092
LOGGER_LEVEL=info
LOG_FOLDER=logs
LOGGER_FILENAME=tb-js-executor-%DATE%.log

3
docker/tb-mqtt-transport.env
View File

@ -4,6 +4,3 @@ ZOOKEEPER_URL=zookeeper:2181
MQTT_BIND_ADDRESS=0.0.0.0
MQTT_BIND_PORT=1883
MQTT_TIMEOUT=10000
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=kafka:9092

2
docker/tb-node.env
View File

@ -2,8 +2,6 @@
ZOOKEEPER_ENABLED=true
ZOOKEEPER_URL=zookeeper:2181
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=kafka:9092
JS_EVALUATOR=remote
TRANSPORT_TYPE=remote
CACHE_TYPE=redis

20
k8s/README.md
View File

@ -40,6 +40,26 @@ Where:
## Running
Execute the following command to deploy thirdparty resources:
`
$ ./k8s-deploy-thirdparty.sh
`
Get list of the running tb-redis pods and verify that all of them are in running state:
`
$ kubectl get pods -l app=tb-redis
`
Execute the following command to create redis cluster:
`
$ kubectl exec -it tb-redis-0 -- redis-cli --cluster create --cluster-replicas 1 $(kubectl get pods -l app=tb-redis -o jsonpath='{range.items[*]}{.status.podIP}:6379 ')
`
Type **'yes'** when prompted.
Execute the following command to deploy resources:
`

1
k8s/k8s-delete-resources.sh
View File

@ -19,3 +19,4 @@ set -e
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl delete -f thingsboard.yml
kubectl delete -f thirdparty.yml

21
k8s/k8s-delete-thirdparty.sh
View File

@ -0,0 +1,21 @@
#!/bin/bash
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl delete -f thirdparty.yml

22
k8s/k8s-deploy-thirdparty.sh
View File

@ -0,0 +1,22 @@
#!/bin/bash
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
kubectl apply -f tb-namespace.yml
kubectl config set-context $(kubectl config current-context) --namespace=thingsboard
kubectl apply -f thirdparty.yml

167
k8s/thingsboard.yml
View File

@ -14,169 +14,6 @@
# limitations under the License.
#
apiVersion: apps/v1
kind: Deployment
metadata:
name: zookeeper
namespace: thingsboard
spec:
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
spec:
containers:
- name: server
imagePullPolicy: Always
image: zookeeper:3.5
ports:
- containerPort: 2181
readinessProbe:
periodSeconds: 5
tcpSocket:
port: 2181
livenessProbe:
periodSeconds: 5
tcpSocket:
port: 2181
env:
- name: ZOO_MY_ID
value: "1"
- name: ZOO_SERVERS
value: "server.1=0.0.0.0:2888:3888;0.0.0.0:2181"
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: zookeeper
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: zookeeper
ports:
- name: zk-port
port: 2181
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-kafka
namespace: thingsboard
spec:
selector:
matchLabels:
app: tb-kafka
template:
metadata:
labels:
app: tb-kafka
spec:
containers:
- name: server
imagePullPolicy: Always
image: wurstmeister/kafka:2.12-2.2.1
ports:
- containerPort: 9092
readinessProbe:
periodSeconds: 20
tcpSocket:
port: 9092
livenessProbe:
periodSeconds: 5
tcpSocket:
port: 9092
env:
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper:2181"
- name: KAFKA_LISTENERS
value: "INSIDE://:9093,OUTSIDE://:9092"
- name: KAFKA_ADVERTISED_LISTENERS
value: "INSIDE://:9093,OUTSIDE://tb-kafka:9092"
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT"
- name: KAFKA_INTER_BROKER_LISTENER_NAME
value: "INSIDE"
- name: KAFKA_CREATE_TOPICS
value: "js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
- name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
value: "false"
- name: KAFKA_LOG_RETENTION_BYTES
value: "1073741824"
- name: KAFKA_LOG_SEGMENT_BYTES
value: "268435456"
- name: KAFKA_LOG_RETENTION_MS
value: "300000"
- name: KAFKA_LOG_CLEANUP_POLICY
value: "delete"
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-kafka
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-kafka
ports:
- name: tb-kafka-port
port: 9092
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: tb-redis
namespace: thingsboard
spec:
selector:
matchLabels:
app: tb-redis
template:
metadata:
labels:
app: tb-redis
spec:
containers:
- name: server
imagePullPolicy: Always
image: redis:4.0
ports:
- containerPort: 6379
readinessProbe:
periodSeconds: 5
tcpSocket:
port: 6379
livenessProbe:
periodSeconds: 5
tcpSocket:
port: 6379
volumeMounts:
- mountPath: /data
name: redis-data
volumes:
- name: redis-data
emptyDir: {}
restartPolicy: Always
---
apiVersion: v1
kind: Service
metadata:
name: tb-redis
namespace: thingsboard
spec:
type: ClusterIP
selector:
app: tb-redis
ports:
- name: tb-redis-port
port: 6379
---
apiVersion: apps/v1
kind: Deployment
metadata:
@ -267,6 +104,10 @@ spec:
value: "redis"
- name: REDIS_HOST
value: "tb-redis"
- name: REDIS_CONNECTION_TYPE
value: "cluster"
- name: REDIS_NODES
value: "tb-redis:6379"
- name: HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE
value: "false"
envFrom:

301
k8s/thirdparty.yml
View File

@ -0,0 +1,301 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: zookeeper
namespace: thingsboard
spec:
serviceName: "zookeeper"
replicas: 3
podManagementPolicy: Parallel
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
spec:
containers:
- name: zookeeper
imagePullPolicy: Always
image: zookeeper:3.5
ports:
- containerPort: 2181
name: client
- containerPort: 2888
name: server
- containerPort: 3888
name: election
readinessProbe:
periodSeconds: 60
tcpSocket:
port: 2181
livenessProbe:
periodSeconds: 60
tcpSocket:
port: 2181
env:
- name: ZOO_SERVERS
value: "server.0=zookeeper-0.zookeeper:2888:3888;2181 server.1=zookeeper-1.zookeeper:2888:3888;2181 server.2=zookeeper-2.zookeeper:2888:3888;2181"
- name: JVMFLAGS
value: "-Dzookeeper.electionPortBindRetry=0"
volumeMounts:
- name: data
mountPath: /data
readOnly: false
initContainers:
- command:
- /bin/bash
- -c
- |-
set -ex;
mkdir -p "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR";
chown "$ZOO_USER:$ZOO_USER" "$ZOO_DATA_LOG_DIR" "$ZOO_DATA_DIR" "$ZOO_CONF_DIR"
if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then
echo $HOSTNAME| rev | cut -d "-" -f1 | rev > "$ZOO_DATA_DIR/myid"
fi
env:
- name: HOSTNAME
valueFrom:
fieldRef:
fieldPath: metadata.name
image: zookeeper:3.5
imagePullPolicy: IfNotPresent
name: zookeeper-init
securityContext:
runAsUser: 0
volumeMounts:
- name: data
mountPath: /data
readOnly: false
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi
---
apiVersion: v1
kind: Service
metadata:
name: zookeeper
namespace: thingsboard
spec:
type: ClusterIP
ports:
- port: 2181
targetPort: 2181
name: client
- port: 2888
targetPort: 2888
name: server
- port: 3888
targetPort: 3888
name: election
selector:
app: zookeeper
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: tb-kafka
namespace: thingsboard
spec:
serviceName: "tb-kafka"
replicas: 3
podManagementPolicy: Parallel
selector:
matchLabels:
app: tb-kafka
template:
metadata:
labels:
app: tb-kafka
spec:
containers:
- name: tb-kafka
imagePullPolicy: Always
image: wurstmeister/kafka:2.12-2.2.1
ports:
- containerPort: 9092
name: kafka-int
readinessProbe:
periodSeconds: 5
timeoutSeconds: 5
tcpSocket:
port: 9092
initialDelaySeconds: 60
livenessProbe:
timeoutSeconds: 5
periodSeconds: 5
tcpSocket:
port: 9092
initialDelaySeconds: 80
env:
- name: BROKER_ID_COMMAND
value: "hostname | cut -d'-' -f3"
- name: KAFKA_ZOOKEEPER_CONNECT
value: "zookeeper:2181"
- name: KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS
value: "60000"
- name: KAFKA_UNCLEAN_LEADER_ELECTION_ENABLE
value: "true"
- name: KAFKA_LISTENERS
value: "INSIDE://:9092"
- name: KAFKA_ADVERTISED_LISTENERS
value: "INSIDE://:9092"
- name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
value: "INSIDE:PLAINTEXT"
- name: KAFKA_INTER_BROKER_LISTENER_NAME
value: "INSIDE"
- name: KAFKA_CONTROLLER_SHUTDOWN_ENABLE
value: "true"
- name: KAFKA_CREATE_TOPICS
value: "js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600"
- name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
value: "false"
- name: KAFKA_LOG_RETENTION_BYTES
value: "1073741824"
- name: KAFKA_LOG_SEGMENT_BYTES
value: "268435456"
- name: KAFKA_LOG_RETENTION_MS
value: "300000"
- name: KAFKA_LOG_CLEANUP_POLICY
value: "delete"
- name: KAFKA_PORT
value: "9092"
- name: KAFKA_LOG_DIRS
value: "/kafka-logs"
volumeMounts:
- name: logs
mountPath: /kafka-logs
subPath: logs
volumeClaimTemplates:
- metadata:
name: logs
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi
---
apiVersion: v1
kind: Service
metadata:
name: tb-kafka
namespace: thingsboard
spec:
type: ClusterIP
ports:
- port: 9092
targetPort: 9092
name: kafka-int
selector:
app: tb-kafka
---
apiVersion: v1
kind: ConfigMap
metadata:
name: tb-redis
namespace: thingsboard
data:
update-node.sh: |
#!/bin/sh
REDIS_NODES="/data/nodes.conf"
sed -i -e "/myself/ s/[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}/${POD_IP}/" ${REDIS_NODES}
exec "$@"
redis.conf: |+
cluster-enabled yes
cluster-require-full-coverage no
cluster-node-timeout 15000
cluster-config-file /data/nodes.conf
cluster-migration-barrier 1
appendonly yes
protected-mode no
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: tb-redis
namespace: thingsboard
spec:
serviceName: server
replicas: 6
selector:
matchLabels:
app: tb-redis
template:
metadata:
labels:
app: tb-redis
spec:
containers:
- name: redis
image: redis:5.0.1-alpine
ports:
- containerPort: 6379
name: client
- containerPort: 16379
name: gossip
command: ["/conf/update-node.sh", "redis-server", "/conf/redis.conf"]
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: conf
mountPath: /conf
readOnly: false
- name: data
mountPath: /data
readOnly: false
volumes:
- name: conf
configMap:
name: tb-redis
defaultMode: 0755
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 100Mi
---
apiVersion: v1
kind: Service
metadata:
name: tb-redis
namespace: thingsboard
spec:
type: ClusterIP
ports:
- port: 6379
targetPort: 6379
name: client
- port: 16379
targetPort: 16379
name: gossip
selector:
app: tb-redis
Write Preview
Loading…
Cancel
Save