From a88d624ffaa90d6808b031f2ad9be998ba4e5417 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 19 Mar 2021 13:51:42 +0200 Subject: [PATCH 01/23] Perfomance improvement for tenant state load --- .../DefaultTbApiUsageStateService.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index d0a3984660..c2549803e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -72,6 +72,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -421,20 +422,33 @@ public class DefaultTbApiUsageStateService extends TbApplicationEventListener tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); - for (Tenant tenant : tenantIterator) { - if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { - log.debug("[{}] Initializing tenant state.", tenant.getId()); - updateLock.lock(); - try { - updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); - log.debug("[{}] Initialized tenant state.", tenant.getId()); - } catch (Exception e) { - log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); - } finally { - updateLock.unlock(); + updateLock.lock(); + try { + ExecutorService tmpInitExecutor = Executors.newWorkStealingPool(20); + try { + PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, 1024); + List> futures = new ArrayList<>(); + for (Tenant tenant : tenantIterator) { + if (!myTenantStates.containsKey(tenant.getId()) && partitionService.resolve(ServiceType.TB_CORE, tenant.getId(), tenant.getId()).isMyPartition()) { + log.debug("[{}] Initializing tenant state.", tenant.getId()); + futures.add(tmpInitExecutor.submit(() -> { + try { + updateTenantState(getOrFetchState(tenant.getId()), tenantProfileCache.get(tenant.getTenantProfileId())); + log.debug("[{}] Initialized tenant state.", tenant.getId()); + } catch (Exception e) { + log.warn("[{}] Failed to initialize tenant API state", tenant.getId(), e); + } + })); + } + } + for (Future future : futures) { + future.get(); } + } finally { + tmpInitExecutor.shutdownNow(); } + } finally { + updateLock.unlock(); } log.info("Initialized tenant states."); } catch (Exception e) { From 0f9cecafbd7d5399506108f6227de2d4dad6ad4a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 19 Mar 2021 15:56:12 +0200 Subject: [PATCH 02/23] Improve data aggregator to send updates on late data --- ui-ngx/src/app/core/api/data-aggregator.ts | 85 ++++++++++++++++++---- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index 1749d89e4e..2444f11dec 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -36,8 +36,56 @@ interface AggData { aggValue: any; } -interface AggregationMap { - [key: string]: Map; +class AggDataMap { + rangeChanged = false; + private minTs = Number.MAX_SAFE_INTEGER; + private map = new Map(); + + set(ts: number, data: AggData) { + if (ts < this.minTs) { + this.rangeChanged = true; + this.minTs = ts; + } + this.map.set(ts, data); + } + + get(ts: number): AggData { + return this.map.get(ts); + } + + delete(ts: number) { + this.map.delete(ts); + } + + forEach(callback: (value: AggData, key: number, map: Map) => void, thisArg?: any) { + this.map.forEach(callback, thisArg); + } + + size(): number { + return this.map.size; + } +} + +class AggregationMap { + aggMap: {[key: string]: AggDataMap} = {}; + + detectRangeChanged(): boolean { + let changed = false; + for (const key of Object.keys(this.aggMap)) { + const aggDataMap = this.aggMap[key]; + if (aggDataMap.rangeChanged) { + changed = true; + aggDataMap.rangeChanged = false; + } + } + return changed; + } + + clearRangeChangedFlags() { + for (const key of Object.keys(this.aggMap)) { + this.aggMap[key].rangeChanged = false; + } + } } declare type AggFunction = (aggData: AggData, value?: any) => void; @@ -170,7 +218,7 @@ export class DataAggregator { updateIntervalScheduledTime = false; } if (update) { - this.aggregationMap = {}; + this.aggregationMap = new AggregationMap(); this.updateAggregatedData(data.data); } else { this.aggregationMap = this.processAggregatedData(data.data); @@ -178,12 +226,17 @@ export class DataAggregator { if (updateIntervalScheduledTime) { this.intervalScheduledTime = this.utils.currentPerfTime(); } + this.aggregationMap.clearRangeChangedFlags(); this.onInterval(history, detectChanges); } else { this.updateAggregatedData(data.data); if (history) { this.intervalScheduledTime = this.utils.currentPerfTime(); this.onInterval(history, detectChanges); + } else { + if (this.aggregationMap.detectRangeChanged()) { + this.onInterval(false, detectChanges, true); + } } } } @@ -203,7 +256,7 @@ export class DataAggregator { } } - private onInterval(history?: boolean, detectChanges?: boolean) { + private onInterval(history?: boolean, detectChanges?: boolean, rangeChanged?: boolean) { const now = this.utils.currentPerfTime(); this.elapsed += now - this.intervalScheduledTime; this.intervalScheduledTime = now; @@ -211,9 +264,10 @@ export class DataAggregator { clearTimeout(this.intervalTimeoutHandle); this.intervalTimeoutHandle = null; } + const intervalTimeout = rangeChanged ? this.aggregationTimeout - this.elapsed : this.aggregationTimeout; if (!history) { const delta = Math.floor(this.elapsed / this.subsTw.aggregation.interval); - if (delta || !this.data) { + if (delta || !this.data || rangeChanged) { const tickTs = delta * this.subsTw.aggregation.interval; if (this.subsTw.quickInterval) { const currentDate = this.getCurrentTime(); @@ -234,7 +288,7 @@ export class DataAggregator { this.updatedData = false; } if (!history) { - this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), this.aggregationTimeout); + this.intervalTimeoutHandle = setTimeout(this.onInterval.bind(this), intervalTimeout); } } @@ -242,8 +296,8 @@ export class DataAggregator { this.tsKeyNames.forEach((key) => { this.dataBuffer[key] = []; }); - for (const key of Object.keys(this.aggregationMap)) { - const aggKeyData = this.aggregationMap[key]; + for (const key of Object.keys(this.aggregationMap.aggMap)) { + const aggKeyData = this.aggregationMap.aggMap[key]; let keyData = this.dataBuffer[key]; aggKeyData.forEach((aggData, aggTimestamp) => { if (aggTimestamp <= this.startTs) { @@ -300,12 +354,12 @@ export class DataAggregator { private processAggregatedData(data: SubscriptionData): AggregationMap { const isCount = this.subsTw.aggregation.type === AggregationType.COUNT; - const aggregationMap: AggregationMap = {}; + const aggregationMap = new AggregationMap(); for (const key of Object.keys(data)) { - let aggKeyData = aggregationMap[key]; + let aggKeyData = aggregationMap.aggMap[key]; if (!aggKeyData) { - aggKeyData = new Map(); - aggregationMap[key] = aggKeyData; + aggKeyData = new AggDataMap(); + aggregationMap.aggMap[key] = aggKeyData; } const keyData = data[key]; keyData.forEach((kvPair) => { @@ -326,10 +380,10 @@ export class DataAggregator { private updateAggregatedData(data: SubscriptionData) { const isCount = this.subsTw.aggregation.type === AggregationType.COUNT; for (const key of Object.keys(data)) { - let aggKeyData = this.aggregationMap[key]; + let aggKeyData = this.aggregationMap.aggMap[key]; if (!aggKeyData) { - aggKeyData = new Map(); - this.aggregationMap[key] = aggKeyData; + aggKeyData = new AggDataMap(); + this.aggregationMap.aggMap[key] = aggKeyData; } const keyData = data[key]; keyData.forEach((kvPair) => { @@ -374,4 +428,3 @@ export class DataAggregator { } } - From e2b455e85133e408c1564136f06e48c749a39217 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 19 Mar 2021 16:04:28 +0200 Subject: [PATCH 03/23] Fix Timescale procedure 'cleanup_timeseries_by_ttl' --- .../data/upgrade/3.2.1/schema_update_ttl.sql | 71 +++++++++++++++++++ .../TimescaleTsDatabaseUpgradeService.java | 4 ++ .../main/resources/sql/schema-timescale.sql | 4 +- 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql diff --git a/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql b/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql new file mode 100644 index 0000000000..94fd2cc00c --- /dev/null +++ b/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql @@ -0,0 +1,71 @@ +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE +tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record uuid; + customer_id_record uuid; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN +OPEN tenant_cursor; +FETCH tenant_cursor INTO tenant_id_record; +WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; +END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; +END IF; +FOR customer_id_record IN +SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; +IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; +ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; +END IF; +END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; +END IF; +END LOOP; +FETCH tenant_cursor INTO tenant_id_record; +END LOOP; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 112ecc3018..92082acf3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -178,7 +178,11 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr } break; case "3.1.1": + break; case "3.2.1": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + } break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index b5808133ec..d0104c45c3 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -91,8 +91,8 @@ $$ DECLARE tenant_cursor CURSOR FOR select tenant.id as tenant_id from tenant; - tenant_id_record varchar; - customer_id_record varchar; + tenant_id_record uuid; + customer_id_record uuid; tenant_ttl bigint; customer_ttl bigint; deleted_for_entities bigint; From ed5f36a4c9a71c6201be8f26974bd96028520937 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 19 Mar 2021 16:48:05 +0200 Subject: [PATCH 04/23] Fix latest subscription reconnect handling --- ui-ngx/src/app/core/api/entity-data-subscription.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index c6f8121919..9c769f3777 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -277,7 +277,11 @@ export class EntityDataSubscription { dataAggregator.reset(newSubsTw); }); } - this.subscriber.setTsOffset(this.subsTw.tsOffset); + if (this.entityDataSubscriptionOptions.type === widgetType.timeseries) { + this.subscriber.setTsOffset(this.subsTw.tsOffset); + } else { + this.subscriber.setTsOffset(this.latestTsOffset); + } targetCommand.query = this.dataCommand.query; this.subscriber.subscriptionCommands = [targetCommand]; } else { From 91116bd71584dbdae9f08963bfef94151e532b35 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 19 Mar 2021 16:55:59 +0200 Subject: [PATCH 05/23] Fix license header --- .../data/upgrade/3.2.1/schema_update_ttl.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql b/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql index 94fd2cc00c..326ca27715 100644 --- a/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql +++ b/application/src/main/data/upgrade/3.2.1/schema_update_ttl.sql @@ -1,3 +1,19 @@ +-- +-- Copyright © 2016-2021 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS From 3c78d614c258bea9df399445055807660446ea20 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 19 Mar 2021 18:45:41 +0200 Subject: [PATCH 06/23] Introduce week intervals (Mon - Sun) and (Sun - Sat) --- .../src/app/shared/models/time/time.models.ts | 35 ++++++++++++++++++- .../assets/locale/locale.constant-en_US.json | 9 +++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index 49ad67dcf1..a686857ed1 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -136,13 +136,16 @@ export enum QuickTimeInterval { DAY_BEFORE_YESTERDAY = 'DAY_BEFORE_YESTERDAY', THIS_DAY_LAST_WEEK = 'THIS_DAY_LAST_WEEK', PREVIOUS_WEEK = 'PREVIOUS_WEEK', + PREVIOUS_WEEK_ISO = 'PREVIOUS_WEEK_ISO', PREVIOUS_MONTH = 'PREVIOUS_MONTH', PREVIOUS_YEAR = 'PREVIOUS_YEAR', CURRENT_HOUR = 'CURRENT_HOUR', CURRENT_DAY = 'CURRENT_DAY', CURRENT_DAY_SO_FAR = 'CURRENT_DAY_SO_FAR', CURRENT_WEEK = 'CURRENT_WEEK', - CURRENT_WEEK_SO_FAR = 'CURRENT_WEEK_SO_WAR', + CURRENT_WEEK_ISO = 'CURRENT_WEEK_ISO', + CURRENT_WEEK_SO_FAR = 'CURRENT_WEEK_SO_FAR', + CURRENT_WEEK_ISO_SO_FAR = 'CURRENT_WEEK_ISO_SO_FAR', CURRENT_MONTH = 'CURRENT_MONTH', CURRENT_MONTH_SO_FAR = 'CURRENT_MONTH_SO_FAR', CURRENT_YEAR = 'CURRENT_YEAR', @@ -154,13 +157,16 @@ export const QuickTimeIntervalTranslationMap = new Map Date: Fri, 19 Mar 2021 19:36:49 +0200 Subject: [PATCH 07/23] Fix data aggregation --- .../timescale/TsKvTimescaleRepository.java | 4 ++-- .../server/dao/sqlts/ts/TsKvRepository.java | 18 +++++++++--------- .../timeseries/CassandraBaseTimeseriesDao.java | 8 ++++---- ui-ngx/src/app/core/api/data-aggregator.ts | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java index b36af1b0e6..981f859ca1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java @@ -33,7 +33,7 @@ public interface TsKvTimescaleRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "AND tskv.ts >= :startTs AND tskv.ts < :endTs") List findAllWithLimit( @Param("entityId") UUID entityId, @Param("entityKey") int key, @@ -44,7 +44,7 @@ public interface TsKvTimescaleRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "AND tskv.ts >= :startTs AND tskv.ts < :endTs") void delete(@Param("entityId") UUID entityId, @Param("entityKey") int key, @Param("startTs") long startTs, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java index 8a59bc9c29..db5534d73e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java @@ -32,7 +32,7 @@ import java.util.concurrent.CompletableFuture; public interface TsKvRepository extends CrudRepository { @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + - "AND tskv.key = :entityKey AND tskv.ts > :startTs AND tskv.ts <= :endTs") + "AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") List findAllWithLimit(@Param("entityId") UUID entityId, @Param("entityKey") int key, @Param("startTs") long startTs, @@ -42,7 +42,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") void delete(@Param("entityId") UUID entityId, @Param("entityKey") int key, @Param("startTs") long startTs, @@ -51,7 +51,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findStringMax(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @@ -63,7 +63,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findNumericMax(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @@ -73,7 +73,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "AND tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findStringMin(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @@ -85,7 +85,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findNumericMin( @Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @@ -98,7 +98,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findCount(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @@ -110,7 +110,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findAvg(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, @@ -122,7 +122,7 @@ public interface TsKvRepository extends CrudRepository :startTs AND tskv.ts <= :endTs") + "WHERE tskv.entityId = :entityId AND tskv.key = :entityKey AND tskv.ts >= :startTs AND tskv.ts < :endTs") CompletableFuture findSum(@Param("entityId") UUID entityId, @Param("entityKey") int entityKey, @Param("startTs") long startTs, diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 2f817634c3..7d09578dd1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -550,8 +550,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.TS_COLUMN + " > ? " - + "AND " + ModelConstants.TS_COLUMN + " <= ?"); + + "AND " + ModelConstants.TS_COLUMN + " >= ? " + + "AND " + ModelConstants.TS_COLUMN + " < ?"); } return deleteStmt; } @@ -740,8 +740,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.TS_COLUMN + " > ? " - + "AND " + ModelConstants.TS_COLUMN + " <= ?" + + "AND " + ModelConstants.TS_COLUMN + " >= ? " + + "AND " + ModelConstants.TS_COLUMN + " < ?" + (type == Aggregation.NONE ? " ORDER BY " + ModelConstants.TS_COLUMN + " " + orderBy + " LIMIT ?" : "")); } } diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index 2444f11dec..2c27e6fcf0 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -300,14 +300,14 @@ export class DataAggregator { const aggKeyData = this.aggregationMap.aggMap[key]; let keyData = this.dataBuffer[key]; aggKeyData.forEach((aggData, aggTimestamp) => { - if (aggTimestamp <= this.startTs) { + if (aggTimestamp < this.startTs) { if (this.subsTw.aggregation.stateData && (!this.lastPrevKvPairData[key] || this.lastPrevKvPairData[key][0] < aggTimestamp)) { this.lastPrevKvPairData[key] = [aggTimestamp, aggData.aggValue]; } aggKeyData.delete(aggTimestamp); this.updatedData = true; - } else if (aggTimestamp <= this.endTs) { + } else if (aggTimestamp < this.endTs) { const kvPair: [number, any] = [aggTimestamp, aggData.aggValue]; keyData.push(kvPair); } From dfcc5ec52e4862bc4ff2a44221e20bf2e0939b6a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Sat, 20 Mar 2021 01:27:19 +0200 Subject: [PATCH 08/23] UI: Improve interval timestamp range calculation. --- ui-ngx/src/app/core/api/data-aggregator.ts | 18 +- .../src/app/core/api/widget-subscription.ts | 13 +- .../entity/entities-table.component.ts | 10 +- .../src/app/shared/models/time/time.models.ts | 239 ++++++++---------- 4 files changed, 119 insertions(+), 161 deletions(-) diff --git a/ui-ngx/src/app/core/api/data-aggregator.ts b/ui-ngx/src/app/core/api/data-aggregator.ts index 2c27e6fcf0..02c1c8649a 100644 --- a/ui-ngx/src/app/core/api/data-aggregator.ts +++ b/ui-ngx/src/app/core/api/data-aggregator.ts @@ -17,10 +17,9 @@ import { SubscriptionData, SubscriptionDataHolder } from '@app/shared/models/telemetry/telemetry.models'; import { AggregationType, calculateIntervalComparisonEndTime, - calculateIntervalEndTime, - calculateIntervalStartTime, + calculateIntervalEndTime, calculateIntervalStartEndTime, getCurrentTime, - getCurrentTimeForComparison, + getCurrentTimeForComparison, getTime, SubscriptionTimewindow } from '@shared/models/time/time.models'; import { UtilsService } from '@core/services/utils.service'; @@ -245,11 +244,12 @@ export class DataAggregator { this.startTs = this.subsTw.startTs + this.subsTw.tsOffset; if (this.subsTw.quickInterval) { if (this.subsTw.timeForComparison === 'previousInterval') { + const startDate = getTime(this.subsTw.startTs, this.subsTw.timezone); const currentDate = getCurrentTime(this.subsTw.timezone); - this.endTs = calculateIntervalComparisonEndTime(this.subsTw.quickInterval, currentDate) + this.subsTw.tsOffset; + this.endTs = calculateIntervalComparisonEndTime(this.subsTw.quickInterval, startDate, currentDate) + this.subsTw.tsOffset; } else { - const currentDate = this.getCurrentTime(); - this.endTs = calculateIntervalEndTime(this.subsTw.quickInterval, currentDate) + this.subsTw.tsOffset; + const startDate = getTime(this.subsTw.startTs, this.subsTw.timezone); + this.endTs = calculateIntervalEndTime(this.subsTw.quickInterval, startDate, this.subsTw.timezone) + this.subsTw.tsOffset; } } else { this.endTs = this.startTs + this.subsTw.aggregation.timeWindow; @@ -270,9 +270,9 @@ export class DataAggregator { if (delta || !this.data || rangeChanged) { const tickTs = delta * this.subsTw.aggregation.interval; if (this.subsTw.quickInterval) { - const currentDate = this.getCurrentTime(); - this.startTs = calculateIntervalStartTime(this.subsTw.quickInterval, currentDate) + this.subsTw.tsOffset; - this.endTs = calculateIntervalEndTime(this.subsTw.quickInterval, currentDate) + this.subsTw.tsOffset; + const startEndTime = calculateIntervalStartEndTime(this.subsTw.quickInterval, this.subsTw.timezone); + this.startTs = startEndTime[0] + this.subsTw.tsOffset; + this.endTs = startEndTime[1] + this.subsTw.tsOffset; } else { this.startTs += tickTs; this.endTs += tickTs; diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 6663735bd5..b74a807729 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -37,12 +37,11 @@ import { } from '@app/shared/models/widget.models'; import { HttpErrorResponse } from '@angular/common/http'; import { - calculateIntervalEndTime, - calculateIntervalStartTime, + calculateIntervalStartEndTime, calculateTsOffset, ComparisonDuration, createSubscriptionTimewindow, createTimewindowForComparison, - getCurrentTime, isHistoryTypeTimewindow, + isHistoryTypeTimewindow, SubscriptionTimewindow, Timewindow, timewindowTypeChanged, toHistoryTimewindow, @@ -1106,11 +1105,9 @@ export class WidgetSubscription implements IWidgetSubscription { this.timeWindow.timezone = this.subscriptionTimewindow.timezone; if (this.subscriptionTimewindow.realtimeWindowMs) { if (this.subscriptionTimewindow.quickInterval) { - const currentDate = getCurrentTime(this.subscriptionTimewindow.timezone); - this.timeWindow.maxTime = calculateIntervalEndTime( - this.subscriptionTimewindow.quickInterval, currentDate) + this.subscriptionTimewindow.tsOffset; - this.timeWindow.minTime = calculateIntervalStartTime( - this.subscriptionTimewindow.quickInterval, currentDate) + this.subscriptionTimewindow.tsOffset; + const startEndTime = calculateIntervalStartEndTime(this.subscriptionTimewindow.quickInterval, this.subscriptionTimewindow.timezone); + this.timeWindow.maxTime = startEndTime[1] + this.subscriptionTimewindow.tsOffset; + this.timeWindow.minTime = startEndTime[0] + this.subscriptionTimewindow.tsOffset; } else { this.timeWindow.maxTime = moment().valueOf() + this.subscriptionTimewindow.tsOffset + this.timeWindow.stDiff; this.timeWindow.minTime = this.timeWindow.maxTime - this.subscriptionTimewindow.realtimeWindowMs; diff --git a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts index c1dd69ecf5..40a6b0cece 100644 --- a/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/entity/entities-table.component.ts @@ -56,9 +56,7 @@ import { DialogService } from '@core/services/dialog.service'; import { AddEntityDialogComponent } from './add-entity-dialog.component'; import { AddEntityDialogData, EntityAction } from '@home/models/entity/entity-component.models'; import { - calculateIntervalEndTime, - calculateIntervalStartTime, - getCurrentTime, + calculateIntervalStartEndTime, HistoryWindowType, Timewindow } from '@shared/models/time/time.models'; @@ -303,9 +301,9 @@ export class EntitiesTableComponent extends PageComponent implements AfterViewIn timePageLink.startTime = currentTime - this.timewindow.history.timewindowMs; timePageLink.endTime = currentTime; } else if (this.timewindow.history.historyType === HistoryWindowType.INTERVAL) { - const currentDate = getCurrentTime(); - timePageLink.startTime = calculateIntervalStartTime(this.timewindow.history.quickInterval, currentDate); - timePageLink.endTime = calculateIntervalEndTime(this.timewindow.history.quickInterval, currentDate); + const startEndTime = calculateIntervalStartEndTime(this.timewindow.history.quickInterval); + timePageLink.startTime = startEndTime[0]; + timePageLink.endTime = startEndTime[1]; } else { timePageLink.startTime = this.timewindow.history.fixedTimewindow.startTimeMs; timePageLink.endTime = this.timewindow.history.fixedTimewindow.endTimeMs; diff --git a/ui-ngx/src/app/shared/models/time/time.models.ts b/ui-ngx/src/app/shared/models/time/time.models.ts index a686857ed1..c9520f09df 100644 --- a/ui-ngx/src/app/shared/models/time/time.models.ts +++ b/ui-ngx/src/app/shared/models/time/time.models.ts @@ -174,19 +174,18 @@ export const QuickTimeIntervalTranslationMap = new Map Date: Mon, 22 Mar 2021 12:42:58 +0200 Subject: [PATCH 09/23] Fix timeseries tests --- .../server/dao/SqlDaoServiceTestSuite.java | 5 +++- .../timeseries/BaseTimeseriesServiceTest.java | 28 +++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java index 4f26e4eb4e..c17c46c8a8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/SqlDaoServiceTestSuite.java @@ -24,7 +24,10 @@ import java.util.Arrays; @RunWith(ClasspathSuite.class) @ClassnameFilters({ - "org.thingsboard.server.dao.service.sql.*SqlTest" + "org.thingsboard.server.dao.service.sql.*SqlTest", + "org.thingsboard.server.dao.service.attributes.sql.*SqlTest", + "org.thingsboard.server.dao.service.event.sql.*SqlTest", + "org.thingsboard.server.dao.service.timeseries.sql.*SqlTest" }) public class SqlDaoServiceTestSuite { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 8ef3e59fcc..b7872fc239 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -143,52 +143,52 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { public void testFindByQueryAscOrder() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 3); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); - saveEntries(deviceId, TS); List queries = new ArrayList<>(); queries.add(new BaseReadTsKvQuery(STRING_KEY, TS - 3, TS, 0, 1000, Aggregation.NONE, "ASC")); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(3, entries.size()); - Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(0)); - Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(1)); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(2)); + Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); + Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); + Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(3, entries.size()); - Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(0)); - Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(1)); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(2)); + Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(0)); + Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); + Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } @Test public void testFindByQueryDescOrder() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 3); saveEntries(deviceId, TS - 2); saveEntries(deviceId, TS - 1); - saveEntries(deviceId, TS); List queries = new ArrayList<>(); queries.add(new BaseReadTsKvQuery(STRING_KEY, TS - 3, TS, 0, 1000, Aggregation.NONE, "DESC")); List entries = tsService.findAll(tenantId, deviceId, queries).get(); Assert.assertEquals(3, entries.size()); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0)); - Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(1)); - Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(2)); + Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(0)); + Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); + Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(2)); EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY)); entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); Assert.assertEquals(3, entries.size()); - Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0)); - Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(1)); - Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(2)); + Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(0)); + Assert.assertEquals(toTsEntry(TS - 2, stringKvEntry), entries.get(1)); + Assert.assertEquals(toTsEntry(TS - 3, stringKvEntry), entries.get(2)); } @Test From 3b6153acc78d1dcdcdda4be33e0e0fdee1f8b061 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 22 Mar 2021 13:10:29 +0200 Subject: [PATCH 10/23] Fix data generation function --- ui-ngx/src/app/core/api/entity-data-subscription.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 9c769f3777..274e896df5 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -799,7 +799,7 @@ export class EntityDataSubscription { } else { prevSeries = [0, 0]; } - for (let time = startTime; time <= endTime && (this.timer || this.history); time += this.frequency) { + for (let time = startTime; time < endTime && (this.timer || this.history); time += this.frequency) { const value = dataKey.func(time, prevSeries[1]); const series: [number, any] = [time, value]; data.push(series); From 3e8824c5294360b4a9c83ccf0d3a278131207f1a Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 23 Mar 2021 13:17:02 +0200 Subject: [PATCH 11/23] UI: Improved dashboard load performance by optimizing widget header template. --- .../home/components/dashboard/dashboard.component.html | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html index a5d50e9507..d893ec848f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard/dashboard.component.html @@ -80,9 +80,9 @@ (mousedown)="widgetMouseDown($event, widget)" (click)="widgetClicked($event, widget)" (contextmenu)="openWidgetContextMenu($event, widget)"> -
-
- +
+
-
Date: Tue, 23 Mar 2021 13:45:53 +0200 Subject: [PATCH 12/23] added logs for oauth2 get clients method --- .../server/controller/OAuth2Controller.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java b/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java index 3d31ba6c0c..6591a1577a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java +++ b/application/src/main/java/org/thingsboard/server/controller/OAuth2Controller.java @@ -19,11 +19,15 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.oauth2.OAuth2ClientsParams; -import org.thingsboard.server.common.data.oauth2.SchemeType; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @@ -31,6 +35,7 @@ import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.utils.MiscUtils; import javax.servlet.http.HttpServletRequest; +import java.util.Enumeration; import java.util.List; @RestController @@ -46,6 +51,14 @@ public class OAuth2Controller extends BaseController { @ResponseBody public List getOAuth2Clients(HttpServletRequest request) throws ThingsboardException { try { + if (log.isDebugEnabled()) { + log.debug("Executing getOAuth2Clients: [{}][{}][{}]", request.getScheme(), request.getServerName(), request.getServerPort()); + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String header = headerNames.nextElement(); + log.debug("Header: {} {}", header, request.getHeader(header)); + } + } return oAuth2Service.getOAuth2Clients(MiscUtils.getScheme(request), MiscUtils.getDomainNameAndPort(request)); } catch (Exception e) { throw handleException(e); From d369e296da2ada00a5c3a3fc8701e50d0a85f8ff Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 24 Mar 2021 11:05:39 +0200 Subject: [PATCH 13/23] Refactor database upgrade services --- .../AbstractSqlTsDatabaseUpgradeService.java | 2 +- .../install/PsqlTsDatabaseUpgradeService.java | 14 +++++++------- .../install/TimescaleTsDatabaseUpgradeService.java | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java index 535a364ff6..5f8b961a46 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/AbstractSqlTsDatabaseUpgradeService.java @@ -50,7 +50,7 @@ public abstract class AbstractSqlTsDatabaseUpgradeService { @Autowired protected InstallScripts installScripts; - protected abstract void loadSql(Connection conn, String fileName); + protected abstract void loadSql(Connection conn, String fileName, String version); protected void loadFunctions(Path sqlFile, Connection conn) throws Exception { String sql = new String(Files.readAllBytes(sqlFile), StandardCharsets.UTF_8); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index fddec0367d..8e6f4859e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -94,7 +94,7 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe log.info("PostgreSQL version is valid!"); if (isOldSchema(conn, 2004003)) { log.info("Load upgrade functions ..."); - loadSql(conn, LOAD_FUNCTIONS_SQL); + loadSql(conn, LOAD_FUNCTIONS_SQL, "2.4.3"); log.info("Updating timeseries schema ..."); executeQuery(conn, CALL_CREATE_PARTITION_TS_KV_TABLE); if (!partitionType.equals("INDEFINITE")) { @@ -179,9 +179,9 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe } log.info("Load TTL functions ..."); - loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "2.4.3"); log.info("Load Drop Partitions functions ..."); - loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3"); executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); @@ -199,9 +199,9 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe case "3.2.1": try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { log.info("Load TTL functions ..."); - loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "2.4.3"); log.info("Load Drop Partitions functions ..."); - loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL, "2.4.3"); executeQuery(conn, "DROP PROCEDURE IF EXISTS cleanup_timeseries_by_ttl(character varying, bigint, bigint);"); executeQuery(conn, "DROP FUNCTION IF EXISTS delete_asset_records_from_ts_kv(character varying, character varying, bigint);"); @@ -244,8 +244,8 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe } @Override - protected void loadSql(Connection conn, String fileName) { - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); + protected void loadSql(Connection conn, String fileName, String version) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName); try { loadFunctions(schemaUpdateFile, conn); log.info("Functions successfully loaded!"); diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index 92082acf3f..417e3f8f1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -89,7 +89,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr log.info("PostgreSQL version is valid!"); if (isOldSchema(conn, 2004003)) { log.info("Load upgrade functions ..."); - loadSql(conn, LOAD_FUNCTIONS_SQL); + loadSql(conn, LOAD_FUNCTIONS_SQL, "2.4.3"); log.info("Updating timescale schema ..."); executeQuery(conn, CALL_CREATE_TS_KV_LATEST_TABLE); executeQuery(conn, CALL_CREATE_NEW_TENANT_TS_KV_TABLE); @@ -165,7 +165,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr } log.info("Load TTL functions ..."); - loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "2.4.3"); executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005000"); log.info("schema timescale updated!"); @@ -181,7 +181,7 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr break; case "3.2.1": try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - loadSql(conn, LOAD_TTL_FUNCTIONS_SQL); + loadSql(conn, LOAD_TTL_FUNCTIONS_SQL, "3.2.1"); } break; default: @@ -205,8 +205,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr } @Override - protected void loadSql(Connection conn, String fileName) { - Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "2.4.3", fileName); + protected void loadSql(Connection conn, String fileName, String version) { + Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, fileName); try { loadFunctions(schemaUpdateFile, conn); log.info("Functions successfully loaded!"); From 1891af54b3e6cdd86666fe54ee29a0649a8eeb20 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 25 Mar 2021 11:10:01 +0200 Subject: [PATCH 14/23] added a fix for string value RPC params: removed redundant escaped characters --- .../server/controller/RpcController.java | 10 +++++++--- .../common/transport/adaptor/JsonConverter.java | 17 +++++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index 269755bd3e..f98bf364df 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -97,9 +97,13 @@ public class RpcController extends BaseController { private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); - RpcRequest cmd = new RpcRequest(rpcRequestBody.get("method").asText(), - jsonMapper.writeValueAsString(rpcRequestBody.get("params"))); - + String requestData; + if (rpcRequestBody.get("params").isTextual()) { + requestData = rpcRequestBody.get("params").asText(); + } else { + requestData = jsonMapper.writeValueAsString(rpcRequestBody.get("params")); + } + RpcRequest cmd = new RpcRequest(rpcRequestBody.get("method").asText(), requestData); if (rpcRequestBody.has("timeout")) { cmd.setTimeout(rpcRequestBody.get("timeout").asLong()); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 4bb1b96b0a..af27f4de1e 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -22,6 +22,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSyntaxException; +import com.google.gson.stream.MalformedJsonException; import org.apache.commons.lang3.math.NumberUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.DataConstants; @@ -108,7 +109,7 @@ public class JsonConverter { public static ClaimDeviceMsg convertToClaimDeviceProto(DeviceId deviceId, String json) { long durationMs = 0L; if (json != null && !json.isEmpty()) { - return convertToClaimDeviceProto(deviceId, new JsonParser().parse(json)); + return convertToClaimDeviceProto(deviceId, JSON_PARSER.parse(json)); } return buildClaimDeviceMsg(deviceId, DataConstants.DEFAULT_SECRET_KEY, durationMs); } @@ -157,7 +158,15 @@ public class JsonConverter { result.addProperty("id", msg.getRequestId()); } result.addProperty("method", msg.getMethodName()); - result.add("params", new JsonParser().parse(msg.getParams())); + try { + result.add("params", JSON_PARSER.parse(msg.getParams())); + } catch (JsonSyntaxException ex) { + if (ex.getCause() instanceof MalformedJsonException) { + result.addProperty("params", msg.getParams()); + } else { + throw ex; + } + } return result; } @@ -406,7 +415,7 @@ public class JsonConverter { public static JsonElement toJson(TransportProtos.ToServerRpcResponseMsg msg) { if (StringUtils.isEmpty(msg.getError())) { - return new JsonParser().parse(msg.getPayload()); + return JSON_PARSER.parse(msg.getPayload()); } else { JsonObject errorMsg = new JsonObject(); errorMsg.addProperty("error", msg.getError()); @@ -562,7 +571,7 @@ public class JsonConverter { } public static TransportProtos.ProvisionDeviceRequestMsg convertToProvisionRequestMsg(String json) { - JsonElement jsonElement = new JsonParser().parse(json); + JsonElement jsonElement = JSON_PARSER.parse(json); if (jsonElement.isJsonObject()) { return buildProvisionRequestMsg(jsonElement.getAsJsonObject()); } else { From ccd316a91b18d78697cba4dd6e7a6251685cbcf4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 25 Mar 2021 12:51:00 +0200 Subject: [PATCH 15/23] UI: Restore data generation condition - fix realtime data generator --- ui-ngx/src/app/core/api/entity-data-subscription.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 274e896df5..9c769f3777 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -799,7 +799,7 @@ export class EntityDataSubscription { } else { prevSeries = [0, 0]; } - for (let time = startTime; time < endTime && (this.timer || this.history); time += this.frequency) { + for (let time = startTime; time <= endTime && (this.timer || this.history); time += this.frequency) { const value = dataKey.func(time, prevSeries[1]); const series: [number, any] = [time, value]; data.push(series); From 1aeb03c0a2dbf71bcb08aa6ed09e67e298ee2c2e Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 25 Mar 2021 13:12:26 +0200 Subject: [PATCH 16/23] UI: Fixed updated device profiles table after added new profiles --- .../device-profiles-table-config.resolver.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts index 3a6862b422..49a256dd55 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts @@ -36,7 +36,6 @@ import { import { DeviceProfileService } from '@core/http/device-profile.service'; import { DeviceProfileComponent } from '@home/components/profile/device-profile.component'; import { DeviceProfileTabsComponent } from './device-profile-tabs.component'; -import { Observable } from 'rxjs'; import { MatDialog } from '@angular/material/dialog'; import { AddDeviceProfileDialogComponent, @@ -138,8 +137,8 @@ export class DeviceProfilesTableConfigResolver implements Resolve { - return this.dialog.open(AddDeviceProfileDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], @@ -147,7 +146,13 @@ export class DeviceProfilesTableConfigResolver implements Resolve { + if (res) { + this.config.table.updateData(); + } + } + ); } setDefaultDeviceProfile($event: Event, deviceProfile: DeviceProfile) { From fa462467fb5d86847d2f42e84bf5814b6d50ab71 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 30 Mar 2021 12:39:12 +0300 Subject: [PATCH 17/23] Fix JAVA_OPTS in config maps to work with Java 11 --- k8s/common/tb-coap-transport-configmap.yml | 10 +++++----- k8s/common/tb-http-transport-configmap.yml | 10 +++++----- k8s/common/tb-mqtt-transport-configmap.yml | 10 +++++----- k8s/common/tb-node-configmap.yml | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/k8s/common/tb-coap-transport-configmap.yml b/k8s/common/tb-coap-transport-configmap.yml index f38ee7ce0e..0f0ea0a8e5 100644 --- a/k8s/common/tb-coap-transport-configmap.yml +++ b/k8s/common/tb-coap-transport-configmap.yml @@ -23,11 +23,11 @@ metadata: name: tb-coap-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-coap-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" - export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" - export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" - export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError" + export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-coap-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" + export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-coap-transport/${TB_SERVICE_ID}-heapdump.bin" + export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" + export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" + export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" export LOG_FILENAME=tb-coap-transport.out export LOADER_PATH=/usr/share/tb-coap-transport/conf logback: | diff --git a/k8s/common/tb-http-transport-configmap.yml b/k8s/common/tb-http-transport-configmap.yml index ffd4c6833c..582df2e5e8 100644 --- a/k8s/common/tb-http-transport-configmap.yml +++ b/k8s/common/tb-http-transport-configmap.yml @@ -23,11 +23,11 @@ metadata: name: tb-http-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-http-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" - export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" - export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" - export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError" + export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-http-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" + export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-http-transport/${TB_SERVICE_ID}-heapdump.bin" + export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" + export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" + export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" export LOG_FILENAME=tb-http-transport.out export LOADER_PATH=/usr/share/tb-http-transport/conf logback: | diff --git a/k8s/common/tb-mqtt-transport-configmap.yml b/k8s/common/tb-mqtt-transport-configmap.yml index 14db242110..0d82938842 100644 --- a/k8s/common/tb-mqtt-transport-configmap.yml +++ b/k8s/common/tb-mqtt-transport-configmap.yml @@ -23,11 +23,11 @@ metadata: name: tb-mqtt-transport-config data: conf: | - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" - export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" - export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" - export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError" + export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" + export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-mqtt-transport/${TB_SERVICE_ID}-heapdump.bin" + export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" + export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" + export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" export LOG_FILENAME=tb-mqtt-transport.out export LOADER_PATH=/usr/share/tb-mqtt-transport/conf logback: | diff --git a/k8s/common/tb-node-configmap.yml b/k8s/common/tb-node-configmap.yml index fd47c9b785..3e796215ab 100644 --- a/k8s/common/tb-node-configmap.yml +++ b/k8s/common/tb-node-configmap.yml @@ -24,11 +24,11 @@ metadata: data: conf: | export JAVA_OPTS="$JAVA_OPTS -Dplatform=deb -Dinstall.data_dir=/usr/share/thingsboard/data" - export JAVA_OPTS="$JAVA_OPTS -Xloggc:/var/log/thingsboard/${TB_SERVICE_ID}/gc.log -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}/heapdump.bin -XX:+PrintGCDetails -XX:+PrintGCDateStamps" - export JAVA_OPTS="$JAVA_OPTS -XX:+PrintHeapAtGC -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10" - export JAVA_OPTS="$JAVA_OPTS -XX:GCLogFileSize=10M -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" - export JAVA_OPTS="$JAVA_OPTS -XX:CMSWaitDuration=10000 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+CMSParallelInitialMarkEnabled" - export JAVA_OPTS="$JAVA_OPTS -XX:+CMSEdenChunksRecordAlways -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+ExitOnOutOfMemoryError" + export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/thingsboard/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" + export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/thingsboard/${TB_SERVICE_ID}-heapdump.bin" + export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" + export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" + export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" export LOG_FILENAME=thingsboard.out export LOADER_PATH=/usr/share/thingsboard/conf,/usr/share/thingsboard/extensions logback: | From 68a73caada309f8bb4a66a1b93c9cdc440b40551 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 30 Mar 2021 15:57:23 +0300 Subject: [PATCH 18/23] Ignore default value for filters with constant key type --- .../rule/engine/profile/AlarmRuleState.java | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java index ab0d9df7c2..928a403fc4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/AlarmRuleState.java @@ -33,10 +33,7 @@ import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpe import org.thingsboard.server.common.data.device.profile.SpecificTimeSchedule; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.FilterPredicateValue; -import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.KeyFilterPredicate; import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.StringFilterPredicate; @@ -275,7 +272,7 @@ class AlarmRuleState { if (value == null) { return false; } - eval = eval && eval(data, value, filter.getPredicate()); + eval = eval && eval(data, value, filter.getPredicate(), filter); } return eval; } @@ -300,33 +297,33 @@ class AlarmRuleState { return value; } - private boolean eval(DataSnapshot data, EntityKeyValue value, KeyFilterPredicate predicate) { + private boolean eval(DataSnapshot data, EntityKeyValue value, KeyFilterPredicate predicate, AlarmConditionFilter filter) { switch (predicate.getType()) { case STRING: - return evalStrPredicate(data, value, (StringFilterPredicate) predicate); + return evalStrPredicate(data, value, (StringFilterPredicate) predicate, filter); case NUMERIC: - return evalNumPredicate(data, value, (NumericFilterPredicate) predicate); + return evalNumPredicate(data, value, (NumericFilterPredicate) predicate, filter); case BOOLEAN: - return evalBoolPredicate(data, value, (BooleanFilterPredicate) predicate); + return evalBoolPredicate(data, value, (BooleanFilterPredicate) predicate, filter); case COMPLEX: - return evalComplexPredicate(data, value, (ComplexFilterPredicate) predicate); + return evalComplexPredicate(data, value, (ComplexFilterPredicate) predicate, filter); default: return false; } } - private boolean evalComplexPredicate(DataSnapshot data, EntityKeyValue ekv, ComplexFilterPredicate predicate) { + private boolean evalComplexPredicate(DataSnapshot data, EntityKeyValue ekv, ComplexFilterPredicate predicate, AlarmConditionFilter filter) { switch (predicate.getOperation()) { case OR: for (KeyFilterPredicate kfp : predicate.getPredicates()) { - if (eval(data, ekv, kfp)) { + if (eval(data, ekv, kfp, filter)) { return true; } } return false; case AND: for (KeyFilterPredicate kfp : predicate.getPredicates()) { - if (!eval(data, ekv, kfp)) { + if (!eval(data, ekv, kfp, filter)) { return false; } } @@ -336,12 +333,15 @@ class AlarmRuleState { } } - private boolean evalBoolPredicate(DataSnapshot data, EntityKeyValue ekv, BooleanFilterPredicate predicate) { + private boolean evalBoolPredicate(DataSnapshot data, EntityKeyValue ekv, BooleanFilterPredicate predicate, AlarmConditionFilter filter) { Boolean val = getBoolValue(ekv); if (val == null) { return false; } - Boolean predicateValue = getPredicateValue(data, predicate.getValue(), AlarmRuleState::getBoolValue); + Boolean predicateValue = getPredicateValue(data, predicate.getValue(), filter, AlarmRuleState::getBoolValue); + if (predicateValue == null) { + return false; + } switch (predicate.getOperation()) { case EQUAL: return val.equals(predicateValue); @@ -352,12 +352,15 @@ class AlarmRuleState { } } - private boolean evalNumPredicate(DataSnapshot data, EntityKeyValue ekv, NumericFilterPredicate predicate) { + private boolean evalNumPredicate(DataSnapshot data, EntityKeyValue ekv, NumericFilterPredicate predicate, AlarmConditionFilter filter) { Double val = getDblValue(ekv); if (val == null) { return false; } - Double predicateValue = getPredicateValue(data, predicate.getValue(), AlarmRuleState::getDblValue); + Double predicateValue = getPredicateValue(data, predicate.getValue(), filter, AlarmRuleState::getDblValue); + if (predicateValue == null) { + return false; + } switch (predicate.getOperation()) { case NOT_EQUAL: return !val.equals(predicateValue); @@ -376,12 +379,15 @@ class AlarmRuleState { } } - private boolean evalStrPredicate(DataSnapshot data, EntityKeyValue ekv, StringFilterPredicate predicate) { + private boolean evalStrPredicate(DataSnapshot data, EntityKeyValue ekv, StringFilterPredicate predicate, AlarmConditionFilter filter) { String val = getStrValue(ekv); if (val == null) { return false; } - String predicateValue = getPredicateValue(data, predicate.getValue(), AlarmRuleState::getStrValue); + String predicateValue = getPredicateValue(data, predicate.getValue(), filter, AlarmRuleState::getStrValue); + if (predicateValue == null) { + return false; + } if (predicate.isIgnoreCase()) { val = val.toLowerCase(); predicateValue = predicateValue.toLowerCase(); @@ -404,7 +410,7 @@ class AlarmRuleState { } } - private T getPredicateValue(DataSnapshot data, FilterPredicateValue value, Function transformFunction) { + private T getPredicateValue(DataSnapshot data, FilterPredicateValue value, AlarmConditionFilter filter, Function transformFunction) { EntityKeyValue ekv = getDynamicPredicateValue(data, value); if (ekv != null) { T result = transformFunction.apply(ekv); @@ -412,7 +418,11 @@ class AlarmRuleState { return result; } } - return value.getDefaultValue(); + if (filter.getKey().getType() != AlarmConditionKeyType.CONSTANT) { + return value.getDefaultValue(); + } else { + return null; + } } private EntityKeyValue getDynamicPredicateValue(DataSnapshot data, FilterPredicateValue value) { From fd3e18f18b28ff5e91de08349fcaa2c416c2a08c Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Tue, 30 Mar 2021 18:28:03 +0300 Subject: [PATCH 19/23] CoAP DTLS support (#4316) * dtls init commit * added fixes after review * fix typo * changed translation for DeviceCredentialsType.X509_CERTIFICATE --- .../src/main/resources/thingsboard.yml | 19 +- common/transport/coap/pom.xml | 4 + .../transport/coap/CoapTransportContext.java | 4 + .../transport/coap/CoapTransportResource.java | 230 ++++++++++-------- .../transport/coap/CoapTransportService.java | 64 ++++- .../coap/TbCoapDtlsCertificateVerifier.java | 161 ++++++++++++ .../TbCoapDtlsSessionInMemoryStorage.java | 55 +++++ .../transport/coap/TbCoapDtlsSessionInfo.java | 35 +++ .../transport/coap/TbCoapDtlsSettings.java | 162 ++++++++++++ .../transport/coap/client/NoSecClient.java | 97 ++++++++ .../coap/client/SecureClientNoAuth.java | 145 +++++++++++ .../coap/client/SecureClientX509.java | 144 +++++++++++ .../mqtt/MqttSslHandlerProvider.java | 3 +- .../transport/mqtt/MqttTransportHandler.java | 2 +- .../transport/auth/SessionInfoCreator.java | 10 +- .../common/transport}/util/SslUtil.java | 3 +- pom.xml | 5 + .../src/main/resources/tb-coap-transport.yml | 18 ++ ui-ngx/src/app/shared/models/device.models.ts | 2 +- 19 files changed, 1050 insertions(+), 113 deletions(-) create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsCertificateVerifier.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInMemoryStorage.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInfo.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSettings.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/NoSecClient.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientNoAuth.java create mode 100644 common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientX509.java rename common/transport/{mqtt/src/main/java/org/thingsboard/server/transport/mqtt => transport-api/src/main/java/org/thingsboard/server/common/transport}/util/SslUtil.java (93%) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 6ac41979d6..ae22aaca51 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -583,7 +583,24 @@ transport: bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" bind_port: "${COAP_BIND_PORT:5683}" timeout: "${COAP_TIMEOUT:10000}" - + dtls: + # Enable/disable DTLS 1.2 support + enabled: "${COAP_DTLS_ENABLED:false}" + # Secure mode. Allowed values: NO_AUTH, X509 + mode: "${COAP_DTLS_SECURE_MODE:NO_AUTH}" + # Path to the key store that holds the certificate + key_store: "${COAP_DTLS_KEY_STORE:coapserver.jks}" + # Password used to access the key store + key_store_password: "${COAP_DTLS_KEY_STORE_PASSWORD:server_ks_password}" + # Password used to access the key + key_password: "${COAP_DTLS_KEY_PASSWORD:server_key_password}" + # Key alias + key_alias: "${COAP_DTLS_KEY_ALIAS:serveralias}" + # Skip certificate validity check for client certificates. + skip_validity_check_for_client_cert: "${COAP_DTLS_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" + x509: + dtls_session_inactivity_timeout: "${TB_COAP_X509_DTLS_SESSION_INACTIVITY_TIMEOUT:86400000}" + dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" security_path_regex: "${SWAGGER_SECURITY_PATH_REGEX:/api.*}" diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index b350e5661a..bd2f027e3e 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -44,6 +44,10 @@ org.eclipse.californium californium-core + + org.eclipse.californium + scandium + org.springframework spring-context-support diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java index bf6c7c994d..83fc56bdc6 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java @@ -46,6 +46,10 @@ public class CoapTransportContext extends TransportContext { @Value("${transport.coap.timeout}") private Long timeout; + @Getter + @Autowired(required = false) + private TbCoapDtlsSettings dtlsSettings; + @Getter @Autowired private JsonCoapAdaptor jsonCoapAdaptor; diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index c1fd5b6a4a..074be3ae47 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -27,6 +27,7 @@ import org.eclipse.californium.core.observe.ObserveRelation; import org.eclipse.californium.core.server.resources.CoapExchange; import org.eclipse.californium.core.server.resources.Resource; import org.eclipse.californium.core.server.resources.ResourceObserver; +import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; @@ -63,15 +64,22 @@ public class CoapTransportResource extends AbstractCoapTransportResource { private static final int FEATURE_TYPE_POSITION = 4; private static final int REQUEST_ID_POSITION = 5; + private static final int FEATURE_TYPE_POSITION_CERTIFICATE_REQUEST = 3; + private static final int REQUEST_ID_POSITION_CERTIFICATE_REQUEST = 4; + private static final String DTLS_SESSION_ID_KEY = "DTLS_SESSION_ID"; + private final ConcurrentMap tokenToSessionIdMap = new ConcurrentHashMap<>(); private final ConcurrentMap tokenToNotificationCounterMap = new ConcurrentHashMap<>(); private final Set rpcSubscriptions = ConcurrentHashMap.newKeySet(); private final Set attributeSubscriptions = ConcurrentHashMap.newKeySet(); - public CoapTransportResource(CoapTransportContext coapTransportContext, String name) { + private ConcurrentMap dtlsSessionIdMap; + + public CoapTransportResource(CoapTransportContext coapTransportContext, ConcurrentMap dtlsSessionIdMap, String name) { super(coapTransportContext, name); this.setObservable(true); // enable observing this.addObserver(new CoapResourceObserver()); + this.dtlsSessionIdMap = dtlsSessionIdMap; // this.setObservable(false); // disable observing // this.setObserveType(CoAP.Type.CON); // configure the notification type to CONs // this.getAttributes().setObservable(); // mark observable in the Link-Format @@ -187,107 +195,132 @@ public class CoapTransportResource extends AbstractCoapTransportResource { Exchange advanced = exchange.advanced(); Request request = advanced.getRequest(); + String dtlsSessionIdStr = request.getSourceContext().get(DTLS_SESSION_ID_KEY); + if (!StringUtils.isEmpty(dtlsSessionIdStr)) { + if (dtlsSessionIdMap != null) { + TbCoapDtlsSessionInfo tbCoapDtlsSessionInfo = dtlsSessionIdMap + .computeIfPresent(dtlsSessionIdStr, (dtlsSessionId, dtlsSessionInfo) -> { + dtlsSessionInfo.setLastActivityTime(System.currentTimeMillis()); + return dtlsSessionInfo; + }); + if (tbCoapDtlsSessionInfo != null) { + processRequest(exchange, type, request, tbCoapDtlsSessionInfo.getSessionInfoProto(), tbCoapDtlsSessionInfo.getDeviceProfile()); + } else { + exchange.respond(CoAP.ResponseCode.UNAUTHORIZED); + } + } else { + processAccessTokenRequest(exchange, type, request); + } + } else { + processAccessTokenRequest(exchange, type, request); + } + } + + private void processAccessTokenRequest(CoapExchange exchange, SessionMsgType type, Request request) { Optional credentials = decodeCredentials(request); if (credentials.isEmpty()) { - exchange.respond(CoAP.ResponseCode.BAD_REQUEST); + exchange.respond(CoAP.ResponseCode.UNAUTHORIZED); return; } - transportService.process(DeviceTransportType.COAP, TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(credentials.get().getCredentialsId()).build(), new CoapDeviceAuthCallback(transportContext, exchange, (sessionInfo, deviceProfile) -> { - UUID sessionId = new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - try { - TransportConfigurationContainer transportConfigurationContainer = getTransportConfigurationContainer(deviceProfile); - CoapTransportAdaptor coapTransportAdaptor = getCoapTransportAdaptor(transportConfigurationContainer.isJsonPayload()); - switch (type) { - case POST_ATTRIBUTES_REQUEST: - transportService.process(sessionInfo, - coapTransportAdaptor.convertToPostAttributes(sessionId, request, - transportConfigurationContainer.getAttributesMsgDescriptor()), - new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - reportActivity(sessionInfo, attributeSubscriptions.contains(sessionId), rpcSubscriptions.contains(sessionId)); - break; - case POST_TELEMETRY_REQUEST: - transportService.process(sessionInfo, - coapTransportAdaptor.convertToPostTelemetry(sessionId, request, - transportConfigurationContainer.getTelemetryMsgDescriptor()), - new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - reportActivity(sessionInfo, attributeSubscriptions.contains(sessionId), rpcSubscriptions.contains(sessionId)); - break; - case CLAIM_REQUEST: - transportService.process(sessionInfo, - coapTransportAdaptor.convertToClaimDevice(sessionId, request, sessionInfo), - new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - break; - case SUBSCRIBE_ATTRIBUTES_REQUEST: - TransportProtos.SessionInfoProto currentAttrSession = tokenToSessionIdMap.get(getTokenFromRequest(request)); - if (currentAttrSession == null) { - attributeSubscriptions.add(sessionId); - registerAsyncCoapSession(exchange, sessionInfo, coapTransportAdaptor, getTokenFromRequest(request)); - transportService.process(sessionInfo, - TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), new CoapNoOpCallback(exchange)); - } - break; - case UNSUBSCRIBE_ATTRIBUTES_REQUEST: - TransportProtos.SessionInfoProto attrSession = lookupAsyncSessionInfo(getTokenFromRequest(request)); - if (attrSession != null) { - UUID attrSessionId = new UUID(attrSession.getSessionIdMSB(), attrSession.getSessionIdLSB()); - attributeSubscriptions.remove(attrSessionId); - transportService.process(attrSession, - TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().setUnsubscribe(true).build(), - new CoapOkCallback(exchange, CoAP.ResponseCode.DELETED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - closeAndDeregister(sessionInfo, sessionId); - } - break; - case SUBSCRIBE_RPC_COMMANDS_REQUEST: - TransportProtos.SessionInfoProto currentRpcSession = tokenToSessionIdMap.get(getTokenFromRequest(request)); - if (currentRpcSession == null) { - rpcSubscriptions.add(sessionId); - registerAsyncCoapSession(exchange, sessionInfo, coapTransportAdaptor, getTokenFromRequest(request)); - transportService.process(sessionInfo, - TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), - new CoapNoOpCallback(exchange)); - } else { - UUID rpcSessionId = new UUID(currentRpcSession.getSessionIdMSB(), currentRpcSession.getSessionIdLSB()); - reportActivity(currentRpcSession, attributeSubscriptions.contains(rpcSessionId), rpcSubscriptions.contains(rpcSessionId)); - } - break; - case UNSUBSCRIBE_RPC_COMMANDS_REQUEST: - TransportProtos.SessionInfoProto rpcSession = lookupAsyncSessionInfo(getTokenFromRequest(request)); - if (rpcSession != null) { - UUID rpcSessionId = new UUID(rpcSession.getSessionIdMSB(), rpcSession.getSessionIdLSB()); - rpcSubscriptions.remove(rpcSessionId); - transportService.process(rpcSession, - TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(), - new CoapOkCallback(exchange, CoAP.ResponseCode.DELETED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - closeAndDeregister(sessionInfo, sessionId); - } - break; - case TO_DEVICE_RPC_RESPONSE: - transportService.process(sessionInfo, - coapTransportAdaptor.convertToDeviceRpcResponse(sessionId, request), - new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); - break; - case TO_SERVER_RPC_REQUEST: - transportService.registerSyncSession(sessionInfo, getCoapSessionListener(exchange, coapTransportAdaptor), transportContext.getTimeout()); - transportService.process(sessionInfo, - coapTransportAdaptor.convertToServerRpcRequest(sessionId, request), - new CoapNoOpCallback(exchange)); - break; - case GET_ATTRIBUTES_REQUEST: - transportService.registerSyncSession(sessionInfo, getCoapSessionListener(exchange, coapTransportAdaptor), transportContext.getTimeout()); - transportService.process(sessionInfo, - coapTransportAdaptor.convertToGetAttributes(sessionId, request), - new CoapNoOpCallback(exchange)); - break; - } - } catch (AdaptorException e) { - log.trace("[{}] Failed to decode message: ", sessionId, e); - exchange.respond(CoAP.ResponseCode.BAD_REQUEST); - } + processRequest(exchange, type, request, sessionInfo, deviceProfile); })); } + private void processRequest(CoapExchange exchange, SessionMsgType type, Request request, TransportProtos.SessionInfoProto sessionInfo, DeviceProfile deviceProfile) { + UUID sessionId = new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); + try { + TransportConfigurationContainer transportConfigurationContainer = getTransportConfigurationContainer(deviceProfile); + CoapTransportAdaptor coapTransportAdaptor = getCoapTransportAdaptor(transportConfigurationContainer.isJsonPayload()); + switch (type) { + case POST_ATTRIBUTES_REQUEST: + transportService.process(sessionInfo, + coapTransportAdaptor.convertToPostAttributes(sessionId, request, + transportConfigurationContainer.getAttributesMsgDescriptor()), + new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + reportActivity(sessionInfo, attributeSubscriptions.contains(sessionId), rpcSubscriptions.contains(sessionId)); + break; + case POST_TELEMETRY_REQUEST: + transportService.process(sessionInfo, + coapTransportAdaptor.convertToPostTelemetry(sessionId, request, + transportConfigurationContainer.getTelemetryMsgDescriptor()), + new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + reportActivity(sessionInfo, attributeSubscriptions.contains(sessionId), rpcSubscriptions.contains(sessionId)); + break; + case CLAIM_REQUEST: + transportService.process(sessionInfo, + coapTransportAdaptor.convertToClaimDevice(sessionId, request, sessionInfo), + new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + break; + case SUBSCRIBE_ATTRIBUTES_REQUEST: + TransportProtos.SessionInfoProto currentAttrSession = tokenToSessionIdMap.get(getTokenFromRequest(request)); + if (currentAttrSession == null) { + attributeSubscriptions.add(sessionId); + registerAsyncCoapSession(exchange, sessionInfo, coapTransportAdaptor, getTokenFromRequest(request)); + transportService.process(sessionInfo, + TransportProtos.SubscribeToAttributeUpdatesMsg.getDefaultInstance(), new CoapNoOpCallback(exchange)); + } + break; + case UNSUBSCRIBE_ATTRIBUTES_REQUEST: + TransportProtos.SessionInfoProto attrSession = lookupAsyncSessionInfo(getTokenFromRequest(request)); + if (attrSession != null) { + UUID attrSessionId = new UUID(attrSession.getSessionIdMSB(), attrSession.getSessionIdLSB()); + attributeSubscriptions.remove(attrSessionId); + transportService.process(attrSession, + TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().setUnsubscribe(true).build(), + new CoapOkCallback(exchange, CoAP.ResponseCode.DELETED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + closeAndDeregister(sessionInfo, sessionId); + } + break; + case SUBSCRIBE_RPC_COMMANDS_REQUEST: + TransportProtos.SessionInfoProto currentRpcSession = tokenToSessionIdMap.get(getTokenFromRequest(request)); + if (currentRpcSession == null) { + rpcSubscriptions.add(sessionId); + registerAsyncCoapSession(exchange, sessionInfo, coapTransportAdaptor, getTokenFromRequest(request)); + transportService.process(sessionInfo, + TransportProtos.SubscribeToRPCMsg.getDefaultInstance(), + new CoapNoOpCallback(exchange)); + } else { + UUID rpcSessionId = new UUID(currentRpcSession.getSessionIdMSB(), currentRpcSession.getSessionIdLSB()); + reportActivity(currentRpcSession, attributeSubscriptions.contains(rpcSessionId), rpcSubscriptions.contains(rpcSessionId)); + } + break; + case UNSUBSCRIBE_RPC_COMMANDS_REQUEST: + TransportProtos.SessionInfoProto rpcSession = lookupAsyncSessionInfo(getTokenFromRequest(request)); + if (rpcSession != null) { + UUID rpcSessionId = new UUID(rpcSession.getSessionIdMSB(), rpcSession.getSessionIdLSB()); + rpcSubscriptions.remove(rpcSessionId); + transportService.process(rpcSession, + TransportProtos.SubscribeToRPCMsg.newBuilder().setUnsubscribe(true).build(), + new CoapOkCallback(exchange, CoAP.ResponseCode.DELETED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + closeAndDeregister(sessionInfo, sessionId); + } + break; + case TO_DEVICE_RPC_RESPONSE: + transportService.process(sessionInfo, + coapTransportAdaptor.convertToDeviceRpcResponse(sessionId, request), + new CoapOkCallback(exchange, CoAP.ResponseCode.CREATED, CoAP.ResponseCode.INTERNAL_SERVER_ERROR)); + break; + case TO_SERVER_RPC_REQUEST: + transportService.registerSyncSession(sessionInfo, getCoapSessionListener(exchange, coapTransportAdaptor), transportContext.getTimeout()); + transportService.process(sessionInfo, + coapTransportAdaptor.convertToServerRpcRequest(sessionId, request), + new CoapNoOpCallback(exchange)); + break; + case GET_ATTRIBUTES_REQUEST: + transportService.registerSyncSession(sessionInfo, getCoapSessionListener(exchange, coapTransportAdaptor), transportContext.getTimeout()); + transportService.process(sessionInfo, + coapTransportAdaptor.convertToGetAttributes(sessionId, request), + new CoapNoOpCallback(exchange)); + break; + } + } catch (AdaptorException e) { + log.trace("[{}] Failed to decode message: ", sessionId, e); + exchange.respond(CoAP.ResponseCode.BAD_REQUEST); + } + } + private TransportProtos.SessionInfoProto lookupAsyncSessionInfo(String token) { tokenToNotificationCounterMap.remove(token); return tokenToSessionIdMap.remove(token); @@ -310,7 +343,7 @@ public class CoapTransportResource extends AbstractCoapTransportResource { private Optional decodeCredentials(Request request) { List uriPath = request.getOptions().getUriPath(); - if (uriPath.size() >= ACCESS_TOKEN_POSITION) { + if (uriPath.size() > ACCESS_TOKEN_POSITION) { return Optional.of(new DeviceTokenCredentials(uriPath.get(ACCESS_TOKEN_POSITION - 1))); } else { return Optional.empty(); @@ -322,8 +355,11 @@ public class CoapTransportResource extends AbstractCoapTransportResource { try { if (uriPath.size() >= FEATURE_TYPE_POSITION) { return Optional.of(FeatureType.valueOf(uriPath.get(FEATURE_TYPE_POSITION - 1).toUpperCase())); - } else if (uriPath.size() == 3 && uriPath.contains(DataConstants.PROVISION)) { - return Optional.of(FeatureType.valueOf(DataConstants.PROVISION.toUpperCase())); + } else if (uriPath.size() >= FEATURE_TYPE_POSITION_CERTIFICATE_REQUEST) { + if (uriPath.contains(DataConstants.PROVISION)) { + return Optional.of(FeatureType.valueOf(DataConstants.PROVISION.toUpperCase())); + } + return Optional.of(FeatureType.valueOf(uriPath.get(FEATURE_TYPE_POSITION_CERTIFICATE_REQUEST - 1).toUpperCase())); } } catch (RuntimeException e) { log.warn("Failed to decode feature type: {}", uriPath); @@ -336,6 +372,8 @@ public class CoapTransportResource extends AbstractCoapTransportResource { try { if (uriPath.size() >= REQUEST_ID_POSITION) { return Optional.of(Integer.valueOf(uriPath.get(REQUEST_ID_POSITION - 1))); + } else { + return Optional.of(Integer.valueOf(uriPath.get(REQUEST_ID_POSITION_CERTIFICATE_REQUEST - 1))); } } catch (RuntimeException e) { log.warn("Failed to decode feature type: {}", uriPath); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 452f661a80..a28019c6c0 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -19,7 +19,10 @@ import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; import org.eclipse.californium.core.network.CoapEndpoint; +import org.eclipse.californium.core.network.config.NetworkConfig; import org.eclipse.californium.core.server.resources.Resource; +import org.eclipse.californium.scandium.DTLSConnector; +import org.eclipse.californium.scandium.config.DtlsConnectorConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -30,6 +33,11 @@ import javax.annotation.PreDestroy; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.util.Random; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; @Service("CoapTransportService") @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") @@ -44,34 +52,53 @@ public class CoapTransportService { @Autowired private CoapTransportContext coapTransportContext; + private TbCoapDtlsCertificateVerifier tbDtlsCertificateVerifier; + private CoapServer server; + private ScheduledExecutorService dtlsSessionsExecutor; + @PostConstruct public void init() throws UnknownHostException { log.info("Starting CoAP transport..."); log.info("Starting CoAP transport server"); this.server = new CoapServer(); + + CoapEndpoint.Builder capEndpointBuilder = new CoapEndpoint.Builder(); + + if (isDtlsEnabled()) { + TbCoapDtlsSettings dtlsSettings = coapTransportContext.getDtlsSettings(); + DtlsConnectorConfig dtlsConnectorConfig = dtlsSettings.dtlsConnectorConfig(); + DTLSConnector connector = new DTLSConnector(dtlsConnectorConfig); + capEndpointBuilder.setConnector(connector); + if (dtlsConnectorConfig.isClientAuthenticationRequired()) { + tbDtlsCertificateVerifier = (TbCoapDtlsCertificateVerifier) dtlsConnectorConfig.getAdvancedCertificateVerifier(); + dtlsSessionsExecutor = Executors.newSingleThreadScheduledExecutor(); + dtlsSessionsExecutor.scheduleAtFixedRate(this::evictTimeoutSessions, new Random().nextInt((int) getDtlsSessionReportTimeout()), getDtlsSessionReportTimeout(), TimeUnit.MILLISECONDS); + } + } else { + InetAddress addr = InetAddress.getByName(coapTransportContext.getHost()); + InetSocketAddress sockAddr = new InetSocketAddress(addr, coapTransportContext.getPort()); + capEndpointBuilder.setInetSocketAddress(sockAddr); + capEndpointBuilder.setNetworkConfig(NetworkConfig.getStandard()); + } + CoapEndpoint coapEndpoint = capEndpointBuilder.build(); + + server.addEndpoint(coapEndpoint); + createResources(); Resource root = this.server.getRoot(); TbCoapServerMessageDeliverer messageDeliverer = new TbCoapServerMessageDeliverer(root); this.server.setMessageDeliverer(messageDeliverer); - InetAddress addr = InetAddress.getByName(coapTransportContext.getHost()); - InetSocketAddress sockAddr = new InetSocketAddress(addr, coapTransportContext.getPort()); - - CoapEndpoint.Builder coapEndpoitBuilder = new CoapEndpoint.Builder(); - coapEndpoitBuilder.setInetSocketAddress(sockAddr); - CoapEndpoint coapEndpoint = coapEndpoitBuilder.build(); - - server.addEndpoint(coapEndpoint); server.start(); log.info("CoAP transport started!"); } private void createResources() { CoapResource api = new CoapResource(API); - api.add(new CoapTransportResource(coapTransportContext, V1)); + api.add(new CoapTransportResource(coapTransportContext, getDtlsSessionsMap(), V1)); CoapResource efento = new CoapResource(EFENTO); CoapEfentoTransportResource efentoMeasurementsTransportResource = new CoapEfentoTransportResource(coapTransportContext, MEASUREMENTS); @@ -81,8 +108,27 @@ public class CoapTransportService { server.add(efento); } + private boolean isDtlsEnabled() { + return coapTransportContext.getDtlsSettings() != null; + } + + private ConcurrentMap getDtlsSessionsMap() { + return tbDtlsCertificateVerifier != null ? tbDtlsCertificateVerifier.getTbCoapDtlsSessionIdsMap() : null; + } + + private void evictTimeoutSessions() { + tbDtlsCertificateVerifier.evictTimeoutSessions(); + } + + private long getDtlsSessionReportTimeout() { + return tbDtlsCertificateVerifier.getDtlsSessionReportTimeout(); + } + @PreDestroy public void shutdown() { + if (dtlsSessionsExecutor != null) { + dtlsSessionsExecutor.shutdownNow(); + } log.info("Stopping CoAP transport!"); this.server.destroy(); log.info("CoAP transport stopped!"); diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsCertificateVerifier.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsCertificateVerifier.java new file mode 100644 index 0000000000..a94ed6caeb --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsCertificateVerifier.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.elements.util.CertPathUtil; +import org.eclipse.californium.scandium.dtls.AlertMessage; +import org.eclipse.californium.scandium.dtls.CertificateMessage; +import org.eclipse.californium.scandium.dtls.CertificateType; +import org.eclipse.californium.scandium.dtls.CertificateVerificationResult; +import org.eclipse.californium.scandium.dtls.ConnectionId; +import org.eclipse.californium.scandium.dtls.DTLSSession; +import org.eclipse.californium.scandium.dtls.HandshakeException; +import org.eclipse.californium.scandium.dtls.HandshakeResultHandler; +import org.eclipse.californium.scandium.dtls.x509.NewAdvancedCertificateVerifier; +import org.eclipse.californium.scandium.util.ServerNames; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.msg.EncryptionUtil; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.auth.SessionInfoCreator; +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; +import org.thingsboard.server.common.transport.util.SslUtil; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; + +import javax.security.auth.x500.X500Principal; +import java.security.cert.CertPath; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateExpiredException; +import java.security.cert.CertificateNotYetValidException; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Data +public class TbCoapDtlsCertificateVerifier implements NewAdvancedCertificateVerifier { + + private final TbCoapDtlsSessionInMemoryStorage tbCoapDtlsSessionInMemoryStorage; + + private TransportService transportService; + private TbServiceInfoProvider serviceInfoProvider; + private boolean skipValidityCheckForClientCert; + + public TbCoapDtlsCertificateVerifier(TransportService transportService, TbServiceInfoProvider serviceInfoProvider, long dtlsSessionInactivityTimeout, long dtlsSessionReportTimeout, boolean skipValidityCheckForClientCert) { + this.transportService = transportService; + this.serviceInfoProvider = serviceInfoProvider; + this.skipValidityCheckForClientCert = skipValidityCheckForClientCert; + this.tbCoapDtlsSessionInMemoryStorage = new TbCoapDtlsSessionInMemoryStorage(dtlsSessionInactivityTimeout, dtlsSessionReportTimeout); + } + + @Override + public List getSupportedCertificateType() { + return Collections.singletonList(CertificateType.X_509); + } + + @Override + public CertificateVerificationResult verifyCertificate(ConnectionId cid, ServerNames serverName, Boolean clientUsage, boolean truncateCertificatePath, CertificateMessage message, DTLSSession session) { + try { + String credentialsBody = null; + CertPath certpath = message.getCertificateChain(); + X509Certificate[] chain = certpath.getCertificates().toArray(new X509Certificate[0]); + for (X509Certificate cert : chain) { + try { + if (!skipValidityCheckForClientCert) { + cert.checkValidity(); + } + String strCert = SslUtil.getCertificateString(cert); + String sha3Hash = EncryptionUtil.getSha3Hash(strCert); + final ValidateDeviceCredentialsResponse[] deviceCredentialsResponse = new ValidateDeviceCredentialsResponse[1]; + CountDownLatch latch = new CountDownLatch(1); + transportService.process(DeviceTransportType.COAP, TransportProtos.ValidateDeviceX509CertRequestMsg.newBuilder().setHash(sha3Hash).build(), + new TransportServiceCallback<>() { + @Override + public void onSuccess(ValidateDeviceCredentialsResponse msg) { + if (!StringUtils.isEmpty(msg.getCredentials())) { + deviceCredentialsResponse[0] = msg; + } + latch.countDown(); + } + + @Override + public void onError(Throwable e) { + log.error(e.getMessage(), e); + latch.countDown(); + } + }); + latch.await(10, TimeUnit.SECONDS); + ValidateDeviceCredentialsResponse msg = deviceCredentialsResponse[0]; + if (msg != null && strCert.equals(msg.getCredentials())) { + credentialsBody = msg.getCredentials(); + DeviceProfile deviceProfile = msg.getDeviceProfile(); + if (msg.hasDeviceInfo() && deviceProfile != null) { + TransportProtos.SessionInfoProto sessionInfoProto = SessionInfoCreator.create(msg, serviceInfoProvider.getServiceId(), UUID.randomUUID()); + tbCoapDtlsSessionInMemoryStorage.put(session.getSessionIdentifier().toString(), new TbCoapDtlsSessionInfo(sessionInfoProto, deviceProfile)); + } + break; + } + } catch (InterruptedException | + CertificateEncodingException | + CertificateExpiredException | + CertificateNotYetValidException e) { + log.error(e.getMessage(), e); + } + } + if (credentialsBody == null) { + AlertMessage alert = new AlertMessage(AlertMessage.AlertLevel.FATAL, AlertMessage.AlertDescription.BAD_CERTIFICATE, + session.getPeer()); + throw new HandshakeException("Certificate chain could not be validated", alert); + } else { + return new CertificateVerificationResult(cid, certpath, null); + } + } catch (HandshakeException e) { + log.trace("Certificate validation failed!", e); + return new CertificateVerificationResult(cid, e, null); + } + } + + @Override + public List getAcceptedIssuers() { + return CertPathUtil.toSubjects(null); + } + + @Override + public void setResultHandler(HandshakeResultHandler resultHandler) { + // empty implementation + } + + public ConcurrentMap getTbCoapDtlsSessionIdsMap() { + return tbCoapDtlsSessionInMemoryStorage.getDtlsSessionIdMap(); + } + + public void evictTimeoutSessions() { + tbCoapDtlsSessionInMemoryStorage.evictTimeoutSessions(); + } + + public long getDtlsSessionReportTimeout() { + return tbCoapDtlsSessionInMemoryStorage.getDtlsSessionReportTimeout(); + } +} \ No newline at end of file diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInMemoryStorage.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInMemoryStorage.java new file mode 100644 index 0000000000..d7dd9c1829 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInMemoryStorage.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +@Data +public class TbCoapDtlsSessionInMemoryStorage { + + private final ConcurrentMap dtlsSessionIdMap = new ConcurrentHashMap<>(); + private long dtlsSessionInactivityTimeout; + private long dtlsSessionReportTimeout; + + + public TbCoapDtlsSessionInMemoryStorage(long dtlsSessionInactivityTimeout, long dtlsSessionReportTimeout) { + this.dtlsSessionInactivityTimeout = dtlsSessionInactivityTimeout; + this.dtlsSessionReportTimeout = dtlsSessionReportTimeout; + } + + public void put(String dtlsSessionId, TbCoapDtlsSessionInfo dtlsSessionInfo) { + log.trace("DTLS session added to in-memory store: [{}] timestamp: [{}]", dtlsSessionId, dtlsSessionInfo.getLastActivityTime()); + dtlsSessionIdMap.putIfAbsent(dtlsSessionId, dtlsSessionInfo); + } + + public void evictTimeoutSessions() { + long expTime = System.currentTimeMillis() - dtlsSessionInactivityTimeout; + dtlsSessionIdMap.entrySet().removeIf(entry -> { + if (entry.getValue().getLastActivityTime() < expTime) { + log.trace("DTLS session was removed from in-memory store: [{}]", entry.getKey()); + return true; + } else { + return false; + } + }); + } + +} \ No newline at end of file diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInfo.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInfo.java new file mode 100644 index 0000000000..452c5eb792 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSessionInfo.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.gen.transport.TransportProtos; + +@Data +public class TbCoapDtlsSessionInfo { + + private TransportProtos.SessionInfoProto sessionInfoProto; + private DeviceProfile deviceProfile; + private long lastActivityTime; + + + public TbCoapDtlsSessionInfo(TransportProtos.SessionInfoProto sessionInfoProto, DeviceProfile deviceProfile) { + this.sessionInfoProto = sessionInfoProto; + this.deviceProfile = deviceProfile; + this.lastActivityTime = System.currentTimeMillis(); + } +} \ No newline at end of file diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSettings.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSettings.java new file mode 100644 index 0000000000..d7cd0b3f39 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/TbCoapDtlsSettings.java @@ -0,0 +1,162 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import com.google.common.io.Resources; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.elements.util.SslContextUtil; +import org.eclipse.californium.scandium.config.DtlsConnectorConfig; +import org.eclipse.californium.scandium.dtls.CertificateType; +import org.eclipse.californium.scandium.dtls.x509.StaticNewAdvancedCertificateVerifier; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.security.GeneralSecurityException; +import java.security.cert.Certificate; +import java.util.Collections; +import java.util.Optional; + +@Slf4j +@ConditionalOnProperty(prefix = "transport.coap.dtls", value = "enabled", havingValue = "true", matchIfMissing = false) +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.coap.enabled}'=='true')") +@Component +public class TbCoapDtlsSettings { + + @Value("${transport.coap.bind_address}") + private String host; + + @Value("${transport.coap.bind_port}") + private Integer port; + + @Value("${transport.coap.dtls.mode}") + private String mode; + + @Value("${transport.coap.dtls.key_store}") + private String keyStoreFile; + + @Value("${transport.coap.dtls.key_store_password}") + private String keyStorePassword; + + @Value("${transport.coap.dtls.key_password}") + private String keyPassword; + + @Value("${transport.coap.dtls.key_alias}") + private String keyAlias; + + @Value("${transport.coap.dtls.skip_validity_check_for_client_cert}") + private boolean skipValidityCheckForClientCert; + + @Value("${transport.coap.dtls.x509.dtls_session_inactivity_timeout}") + private long dtlsSessionInactivityTimeout; + + @Value("${transport.coap.dtls.x509.dtls_session_report_timeout}") + private long dtlsSessionReportTimeout; + + @Autowired + private TransportService transportService; + + @Autowired + private TbServiceInfoProvider serviceInfoProvider; + + public DtlsConnectorConfig dtlsConnectorConfig() throws UnknownHostException { + Optional securityModeOpt = SecurityMode.parse(mode); + if (securityModeOpt.isEmpty()) { + log.warn("Incorrect configuration of securityMode {}", mode); + throw new RuntimeException("Failed to parse mode property: " + mode + "!"); + } else { + DtlsConnectorConfig.Builder configBuilder = new DtlsConnectorConfig.Builder(); + configBuilder.setAddress(getInetSocketAddress()); + String keyStoreFilePath = Resources.getResource(keyStoreFile).getPath(); + SslContextUtil.Credentials serverCredentials = loadServerCredentials(keyStoreFilePath); + SecurityMode securityMode = securityModeOpt.get(); + if (securityMode.equals(SecurityMode.NO_AUTH)) { + configBuilder.setClientAuthenticationRequired(false); + configBuilder.setServerOnly(true); + } else { + configBuilder.setAdvancedCertificateVerifier( + new TbCoapDtlsCertificateVerifier( + transportService, + serviceInfoProvider, + dtlsSessionInactivityTimeout, + dtlsSessionReportTimeout, + skipValidityCheckForClientCert + ) + ); + } + configBuilder.setIdentity(serverCredentials.getPrivateKey(), serverCredentials.getCertificateChain(), + Collections.singletonList(CertificateType.X_509)); + return configBuilder.build(); + } + } + + private SslContextUtil.Credentials loadServerCredentials(String keyStoreFilePath) { + try { + return SslContextUtil.loadCredentials(keyStoreFilePath, keyAlias, keyStorePassword.toCharArray(), + keyPassword.toCharArray()); + } catch (GeneralSecurityException | IOException e) { + throw new RuntimeException("Failed to load serverCredentials due to: ", e); + } + } + + private void loadTrustedCertificates(DtlsConnectorConfig.Builder config, String keyStoreFilePath) { + StaticNewAdvancedCertificateVerifier.Builder trustBuilder = StaticNewAdvancedCertificateVerifier.builder(); + try { + Certificate[] trustedCertificates = SslContextUtil.loadTrustedCertificates( + keyStoreFilePath, keyAlias, + keyStorePassword.toCharArray()); + trustBuilder.setTrustedCertificates(trustedCertificates); + if (trustBuilder.hasTrusts()) { + config.setAdvancedCertificateVerifier(trustBuilder.build()); + } + } catch (GeneralSecurityException | IOException e) { + throw new RuntimeException("Failed to load trusted certificates due to: ", e); + } + } + + private InetSocketAddress getInetSocketAddress() throws UnknownHostException { + InetAddress addr = InetAddress.getByName(host); + return new InetSocketAddress(addr, port); + } + + private enum SecurityMode { + X509, + NO_AUTH; + + static Optional parse(String name) { + SecurityMode mode = null; + if (name != null) { + for (SecurityMode securityMode : SecurityMode.values()) { + if (securityMode.name().equalsIgnoreCase(name)) { + mode = securityMode; + break; + } + } + } + return Optional.ofNullable(mode); + } + + } + +} \ No newline at end of file diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/NoSecClient.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/NoSecClient.java new file mode 100644 index 0000000000..f9a31d0513 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/NoSecClient.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap.client; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.Utils; +import org.eclipse.californium.elements.DtlsEndpointContext; +import org.eclipse.californium.elements.EndpointContext; +import org.eclipse.californium.elements.exception.ConnectorException; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.Principal; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class NoSecClient { + + private ExecutorService executor = Executors.newFixedThreadPool(1); + private CoapClient coapClient; + + public NoSecClient(String host, int port, String accessToken, String clientKeys, String sharedKeys) throws URISyntaxException { + URI uri = new URI(getFutureUrl(host, port, accessToken, clientKeys, sharedKeys)); + this.coapClient = new CoapClient(uri); + } + + public void test() { + executor.submit(() -> { + try { + while (!Thread.interrupted()) { + CoapResponse response = null; + try { + response = coapClient.get(); + } catch (ConnectorException | IOException e) { + System.err.println("Error occurred while sending request: " + e); + System.exit(-1); + } + if (response != null) { + + System.out.println(response.getCode() + " - " + response.getCode().name()); + System.out.println(response.getOptions()); + System.out.println(response.getResponseText()); + System.out.println(); + System.out.println("ADVANCED:"); + EndpointContext context = response.advanced().getSourceContext(); + Principal identity = context.getPeerIdentity(); + if (identity != null) { + System.out.println(context.getPeerIdentity()); + } else { + System.out.println("anonymous"); + } + System.out.println(context.get(DtlsEndpointContext.KEY_CIPHER)); + System.out.println(Utils.prettyPrint(response)); + } else { + System.out.println("No response received."); + } + Thread.sleep(5000); + } + } catch (Exception e) { + System.out.println("Error occurred while sending COAP requests."); + } + }); + } + + private String getFutureUrl(String host, Integer port, String accessToken, String clientKeys, String sharedKeys) { + return "coap://" + host + ":" + port + "/api/v1/" + accessToken + "/attributes?clientKeys=" + clientKeys + "&sharedKeys=" + sharedKeys; + } + + public static void main(String[] args) throws URISyntaxException { + System.out.println("Usage: java -cp ... org.thingsboard.server.transport.coap.client.NoSecClient " + + "host port accessToken clientKeys sharedKeys"); + + String host = args[0]; + int port = Integer.parseInt(args[1]); + String accessToken = args[2]; + String clientKeys = args[3]; + String sharedKeys = args[4]; + + NoSecClient client = new NoSecClient(host, port, accessToken, clientKeys, sharedKeys); + client.test(); + } +} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientNoAuth.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientNoAuth.java new file mode 100644 index 0000000000..7bbb1f55cf --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientNoAuth.java @@ -0,0 +1,145 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap.client; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.Utils; +import org.eclipse.californium.core.network.CoapEndpoint; +import org.eclipse.californium.elements.DtlsEndpointContext; +import org.eclipse.californium.elements.EndpointContext; +import org.eclipse.californium.elements.exception.ConnectorException; +import org.eclipse.californium.elements.util.SslContextUtil; +import org.eclipse.californium.scandium.DTLSConnector; +import org.eclipse.californium.scandium.config.DtlsConnectorConfig; +import org.eclipse.californium.scandium.dtls.CertificateType; +import org.eclipse.californium.scandium.dtls.x509.StaticNewAdvancedCertificateVerifier; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.GeneralSecurityException; +import java.security.Principal; +import java.security.cert.Certificate; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class SecureClientNoAuth { + + private final DTLSConnector dtlsConnector; + private ExecutorService executor = Executors.newFixedThreadPool(1); + private CoapClient coapClient; + + public SecureClientNoAuth(DTLSConnector dtlsConnector, String host, int port, String accessToken, String clientKeys, String sharedKeys) throws URISyntaxException { + this.dtlsConnector = dtlsConnector; + this.coapClient = getCoapClient(host, port, accessToken, clientKeys, sharedKeys); + } + + public void test() { + executor.submit(() -> { + try { + while (!Thread.interrupted()) { + CoapResponse response = null; + try { + response = coapClient.get(); + } catch (ConnectorException | IOException e) { + System.err.println("Error occurred while sending request: " + e); + System.exit(-1); + } + if (response != null) { + + System.out.println(response.getCode() + " - " + response.getCode().name()); + System.out.println(response.getOptions()); + System.out.println(response.getResponseText()); + System.out.println(); + System.out.println("ADVANCED:"); + EndpointContext context = response.advanced().getSourceContext(); + Principal identity = context.getPeerIdentity(); + if (identity != null) { + System.out.println(context.getPeerIdentity()); + } else { + System.out.println("anonymous"); + } + System.out.println(context.get(DtlsEndpointContext.KEY_CIPHER)); + System.out.println(Utils.prettyPrint(response)); + } else { + System.out.println("No response received."); + } + Thread.sleep(5000); + } + } catch (Exception e) { + System.out.println("Error occurred while sending COAP requests."); + } + }); + } + + private CoapClient getCoapClient(String host, Integer port, String accessToken, String clientKeys, String sharedKeys) throws URISyntaxException { + URI uri = new URI(getFutureUrl(host, port, accessToken, clientKeys, sharedKeys)); + CoapClient client = new CoapClient(uri); + CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); + builder.setConnector(dtlsConnector); + + client.setEndpoint(builder.build()); + return client; + } + + private String getFutureUrl(String host, Integer port, String accessToken, String clientKeys, String sharedKeys) { + return "coaps://" + host + ":" + port + "/api/v1/" + accessToken + "/attributes?clientKeys=" + clientKeys + "&sharedKeys=" + sharedKeys; + } + + public static void main(String[] args) throws URISyntaxException { + System.out.println("Usage: java -cp ... org.thingsboard.server.transport.coap.client.SecureClientNoAuth " + + "host port accessToken keyStoreUriPath keyStoreAlias trustedAliasPattern clientKeys sharedKeys"); + + String host = args[0]; + int port = Integer.parseInt(args[1]); + String accessToken = args[2]; + String clientKeys = args[7]; + String sharedKeys = args[8]; + + String keyStoreUriPath = args[3]; + String keyStoreAlias = args[4]; + String trustedAliasPattern = args[5]; + String keyStorePassword = args[6]; + + + DtlsConnectorConfig.Builder builder = new DtlsConnectorConfig.Builder(); + setupCredentials(builder, keyStoreUriPath, keyStoreAlias, trustedAliasPattern, keyStorePassword); + DTLSConnector dtlsConnector = new DTLSConnector(builder.build()); + SecureClientNoAuth client = new SecureClientNoAuth(dtlsConnector, host, port, accessToken, clientKeys, sharedKeys); + client.test(); + } + + private static void setupCredentials(DtlsConnectorConfig.Builder config, String keyStoreUriPath, String keyStoreAlias, String trustedAliasPattern, String keyStorePassword) { + StaticNewAdvancedCertificateVerifier.Builder trustBuilder = StaticNewAdvancedCertificateVerifier.builder(); + try { + SslContextUtil.Credentials serverCredentials = SslContextUtil.loadCredentials( + keyStoreUriPath, keyStoreAlias, keyStorePassword.toCharArray(), keyStorePassword.toCharArray()); + Certificate[] trustedCertificates = SslContextUtil.loadTrustedCertificates( + keyStoreUriPath, trustedAliasPattern, keyStorePassword.toCharArray()); + trustBuilder.setTrustedCertificates(trustedCertificates); + config.setAdvancedCertificateVerifier(trustBuilder.build()); + config.setIdentity(serverCredentials.getPrivateKey(), serverCredentials.getCertificateChain(), Collections.singletonList(CertificateType.X_509)); + } catch (GeneralSecurityException e) { + System.err.println("certificates are invalid!"); + throw new IllegalArgumentException(e.getMessage()); + } catch (IOException e) { + System.err.println("certificates are missing!"); + throw new IllegalArgumentException(e.getMessage()); + } + } +} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientX509.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientX509.java new file mode 100644 index 0000000000..31dd628b40 --- /dev/null +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/client/SecureClientX509.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap.client; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.Utils; +import org.eclipse.californium.core.network.CoapEndpoint; +import org.eclipse.californium.elements.DtlsEndpointContext; +import org.eclipse.californium.elements.EndpointContext; +import org.eclipse.californium.elements.exception.ConnectorException; +import org.eclipse.californium.elements.util.SslContextUtil; +import org.eclipse.californium.scandium.DTLSConnector; +import org.eclipse.californium.scandium.config.DtlsConnectorConfig; +import org.eclipse.californium.scandium.dtls.CertificateType; +import org.eclipse.californium.scandium.dtls.x509.StaticNewAdvancedCertificateVerifier; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.GeneralSecurityException; +import java.security.Principal; +import java.security.cert.Certificate; +import java.util.Collections; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class SecureClientX509 { + + private final DTLSConnector dtlsConnector; + private ExecutorService executor = Executors.newFixedThreadPool(1); + private CoapClient coapClient; + + public SecureClientX509(DTLSConnector dtlsConnector, String host, int port, String clientKeys, String sharedKeys) throws URISyntaxException { + this.dtlsConnector = dtlsConnector; + this.coapClient = getCoapClient(host, port, clientKeys, sharedKeys); + } + + public void test() { + executor.submit(() -> { + try { + while (!Thread.interrupted()) { + CoapResponse response = null; + try { + response = coapClient.get(); + } catch (ConnectorException | IOException e) { + System.err.println("Error occurred while sending request: " + e); + System.exit(-1); + } + if (response != null) { + + System.out.println(response.getCode() + " - " + response.getCode().name()); + System.out.println(response.getOptions()); + System.out.println(response.getResponseText()); + System.out.println(); + System.out.println("ADVANCED:"); + EndpointContext context = response.advanced().getSourceContext(); + Principal identity = context.getPeerIdentity(); + if (identity != null) { + System.out.println(context.getPeerIdentity()); + } else { + System.out.println("anonymous"); + } + System.out.println(context.get(DtlsEndpointContext.KEY_CIPHER)); + System.out.println(Utils.prettyPrint(response)); + } else { + System.out.println("No response received."); + } + Thread.sleep(5000); + } + } catch (Exception e) { + System.out.println("Error occurred while sending COAP requests."); + } + }); + } + + private CoapClient getCoapClient(String host, Integer port, String clientKeys, String sharedKeys) throws URISyntaxException { + URI uri = new URI(getFutureUrl(host, port, clientKeys, sharedKeys)); + CoapClient client = new CoapClient(uri); + CoapEndpoint.Builder builder = new CoapEndpoint.Builder(); + builder.setConnector(dtlsConnector); + + client.setEndpoint(builder.build()); + return client; + } + + private String getFutureUrl(String host, Integer port, String clientKeys, String sharedKeys) { + return "coaps://" + host + ":" + port + "/api/v1/attributes?clientKeys=" + clientKeys + "&sharedKeys=" + sharedKeys; + } + + public static void main(String[] args) throws URISyntaxException { + System.out.println("Usage: java -cp ... org.thingsboard.server.transport.coap.client.SecureClientX509 " + + "host port keyStoreUriPath keyStoreAlias trustedAliasPattern clientKeys sharedKeys"); + + String host = args[0]; + int port = Integer.parseInt(args[1]); + String clientKeys = args[6]; + String sharedKeys = args[7]; + + String keyStoreUriPath = args[2]; + String keyStoreAlias = args[3]; + String trustedAliasPattern = args[4]; + String keyStorePassword = args[5]; + + + DtlsConnectorConfig.Builder builder = new DtlsConnectorConfig.Builder(); + setupCredentials(builder, keyStoreUriPath, keyStoreAlias, trustedAliasPattern, keyStorePassword); + DTLSConnector dtlsConnector = new DTLSConnector(builder.build()); + SecureClientX509 client = new SecureClientX509(dtlsConnector, host, port, clientKeys, sharedKeys); + client.test(); + } + + private static void setupCredentials(DtlsConnectorConfig.Builder config, String keyStoreUriPath, String keyStoreAlias, String trustedAliasPattern, String keyStorePassword) { + StaticNewAdvancedCertificateVerifier.Builder trustBuilder = StaticNewAdvancedCertificateVerifier.builder(); + try { + SslContextUtil.Credentials serverCredentials = SslContextUtil.loadCredentials( + keyStoreUriPath, keyStoreAlias, keyStorePassword.toCharArray(), keyStorePassword.toCharArray()); + Certificate[] trustedCertificates = SslContextUtil.loadTrustedCertificates( + keyStoreUriPath, trustedAliasPattern, keyStorePassword.toCharArray()); + trustBuilder.setTrustedCertificates(trustedCertificates); + config.setAdvancedCertificateVerifier(trustBuilder.build()); + config.setIdentity(serverCredentials.getPrivateKey(), serverCredentials.getCertificateChain(), Collections.singletonList(CertificateType.X_509)); + } catch (GeneralSecurityException e) { + System.err.println("certificates are invalid!"); + throw new IllegalArgumentException(e.getMessage()); + } catch (IOException e) { + System.err.println("certificates are missing!"); + throw new IllegalArgumentException(e.getMessage()); + } + } +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java index c2cf3686e9..1c0801c64a 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.mqtt.util.SslUtil; +import org.thingsboard.server.common.transport.util.SslUtil; import javax.net.ssl.KeyManager; import javax.net.ssl.KeyManagerFactory; @@ -41,7 +41,6 @@ import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; import java.io.File; import java.io.FileInputStream; -import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.security.KeyStore; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 06a8dcacdc..6056aa293d 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -66,7 +66,7 @@ import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor; import org.thingsboard.server.transport.mqtt.session.DeviceSessionCtx; import org.thingsboard.server.transport.mqtt.session.GatewaySessionHandler; import org.thingsboard.server.transport.mqtt.session.MqttTopicMatcher; -import org.thingsboard.server.transport.mqtt.util.SslUtil; +import org.thingsboard.server.common.transport.util.SslUtil; import javax.net.ssl.SSLPeerUnverifiedException; import java.security.cert.Certificate; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/auth/SessionInfoCreator.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/auth/SessionInfoCreator.java index ab18b930f9..b175ca8580 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/auth/SessionInfoCreator.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/auth/SessionInfoCreator.java @@ -25,7 +25,15 @@ import java.util.UUID; public class SessionInfoCreator { public static TransportProtos.SessionInfoProto create(ValidateDeviceCredentialsResponse msg, TransportContext context, UUID sessionId) { - return TransportProtos.SessionInfoProto.newBuilder().setNodeId(context.getNodeId()) + return getSessionInfoProto(msg, context.getNodeId(), sessionId); + } + + public static TransportProtos.SessionInfoProto create(ValidateDeviceCredentialsResponse msg, String nodeId, UUID sessionId) { + return getSessionInfoProto(msg, nodeId, sessionId); + } + + private static TransportProtos.SessionInfoProto getSessionInfoProto(ValidateDeviceCredentialsResponse msg, String nodeId, UUID sessionId) { + return TransportProtos.SessionInfoProto.newBuilder().setNodeId(nodeId) .setSessionIdMSB(sessionId.getMostSignificantBits()) .setSessionIdLSB(sessionId.getLeastSignificantBits()) .setDeviceIdMSB(msg.getDeviceInfo().getDeviceId().getId().getMostSignificantBits()) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/SslUtil.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/SslUtil.java similarity index 93% rename from common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/SslUtil.java rename to common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/SslUtil.java index f376077b84..77e4045655 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/util/SslUtil.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/SslUtil.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.mqtt.util; +package org.thingsboard.server.common.transport.util; import lombok.extern.slf4j.Slf4j; import org.springframework.util.Base64Utils; import org.thingsboard.server.common.msg.EncryptionUtil; -import java.io.IOException; import java.security.cert.Certificate; import java.security.cert.CertificateEncodingException; diff --git a/pom.xml b/pom.xml index d1edef19fb..d9af0abf12 100755 --- a/pom.xml +++ b/pom.xml @@ -1152,6 +1152,11 @@ californium-core ${californium.version} + + org.eclipse.californium + scandium + ${californium.version} + com.google.code.gson gson diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index a9fe673b28..0ffcd27f21 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -46,6 +46,24 @@ transport: bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" bind_port: "${COAP_BIND_PORT:5683}" timeout: "${COAP_TIMEOUT:10000}" + dtls: + # Enable/disable DTLS 1.2 support + enabled: "${COAP_DTLS_ENABLED:false}" + # Secure mode. Allowed values: NO_AUTH, X509 + mode: "${COAP_DTLS_SECURE_MODE:NO_AUTH}" + # Path to the key store that holds the certificate + key_store: "${COAP_DTLS_KEY_STORE:coapserver.jks}" + # Password used to access the key store + key_store_password: "${COAP_DTLS_KEY_STORE_PASSWORD:server_ks_password}" + # Password used to access the key + key_password: "${COAP_DTLS_KEY_PASSWORD:server_key_password}" + # Key alias + key_alias: "${COAP_DTLS_KEY_ALIAS:serveralias}" + # Skip certificate validity check for client certificates. + skip_validity_check_for_client_cert: "${COAP_DTLS_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" + x509: + dtls_session_inactivity_timeout: "${TB_COAP_X509_DTLS_SESSION_INACTIVITY_TIMEOUT:86400000}" + dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 0373f79737..f3ae969911 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -518,7 +518,7 @@ export enum DeviceCredentialsType { export const credentialTypeNames = new Map( [ [DeviceCredentialsType.ACCESS_TOKEN, 'Access token'], - [DeviceCredentialsType.X509_CERTIFICATE, 'MQTT X.509'], + [DeviceCredentialsType.X509_CERTIFICATE, 'X.509'], [DeviceCredentialsType.MQTT_BASIC, 'MQTT Basic'] ] ); From 593f95a7af809c1f5dc4917aa123363a0f0971a8 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 30 Mar 2021 18:29:55 +0300 Subject: [PATCH 20/23] Provide additional validation for entities (#4326) * Provide additional validation for entities * Refactor * Create test for NoXssValidator * Refactor dependencies --- common/data/pom.xml | 8 + .../server/common/data/AdminSettings.java | 4 +- .../server/common/data/ContactBased.java | 11 +- .../server/common/data/Customer.java | 6 +- .../server/common/data/Device.java | 4 + .../server/common/data/DeviceProfile.java | 7 + .../server/common/data/EntityView.java | 3 + .../server/common/data/Tenant.java | 5 +- .../server/common/data/TenantProfile.java | 3 + .../thingsboard/server/common/data/User.java | 4 +- .../server/common/data/asset/Asset.java | 10 +- .../data/device/profile/AlarmCondition.java | 4 +- .../device/profile/AlarmConditionFilter.java | 6 + .../profile/AlarmConditionFilterKey.java | 2 + .../common/data/device/profile/AlarmRule.java | 5 + .../device/profile/DeviceProfileAlarm.java | 5 + .../device/profile/DeviceProfileData.java | 2 + .../common/data/query/DynamicValue.java | 2 + .../data/query/FilterPredicateValue.java | 6 + .../data/query/StringFilterPredicate.java | 3 + .../server/common/data/rule/RuleChain.java | 2 + .../server/common/data/validation/NoXss.java | 34 ++++ dao/pom.xml | 13 ++ .../server/dao/service/DataValidator.java | 38 ++++ .../server/dao/service/NoXssValidator.java | 57 ++++++ dao/src/main/resources/xss-policy.xml | 162 ++++++++++++++++++ .../dao/service/NoXssValidatorTest.java | 52 ++++++ dao/src/test/resources/xss-policy.xml | 162 ++++++++++++++++++ pom.xml | 41 +++++ 29 files changed, 650 insertions(+), 11 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/validation/NoXss.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java create mode 100644 dao/src/main/resources/xss-policy.xml create mode 100644 dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java create mode 100644 dao/src/test/resources/xss-policy.xml diff --git a/common/data/pom.xml b/common/data/pom.xml index ea4d8cb2d8..3067eba918 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -36,6 +36,14 @@ + + javax.validation + validation-api + + + org.owasp.antisamy + antisamy + org.slf4j slf4j-api diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 9389a2f3d6..356af5dab7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -18,11 +18,13 @@ package org.thingsboard.server.common.data; import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.validation.NoXss; public class AdminSettings extends BaseData { private static final long serialVersionUID = -7670322981725511892L; - + + @NoXss private String key; private transient JsonNode jsonValue; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ContactBased.java b/common/data/src/main/java/org/thingsboard/server/common/data/ContactBased.java index 9af8ddb736..a333591e53 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ContactBased.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ContactBased.java @@ -17,19 +17,28 @@ package org.thingsboard.server.common.data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.validation.NoXss; @EqualsAndHashCode(callSuper = true) public abstract class ContactBased extends SearchTextBasedWithAdditionalInfo implements HasName { private static final long serialVersionUID = 5047448057830660988L; - + + @NoXss protected String country; + @NoXss protected String state; + @NoXss protected String city; + @NoXss protected String address; + @NoXss protected String address2; + @NoXss protected String zip; + @NoXss protected String phone; + @NoXss protected String email; public ContactBased() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index e40ab84925..f6f49bb33b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,13 +20,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; - -import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.validation.NoXss; public class Customer extends ContactBased implements HasTenantId { private static final long serialVersionUID = -1599722990298929275L; - + + @NoXss private String title; private TenantId tenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index aef34d6306..bdda77216d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.NoXss; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -37,8 +38,11 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen private TenantId tenantId; private CustomerId customerId; + @NoXss private String name; + @NoXss private String type; + @NoXss private String label; private DeviceProfileId deviceProfileId; private transient DeviceData deviceData; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 8b24cefa71..44c1c4b0ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -24,7 +24,9 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.NoXss; +import javax.validation.Valid; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -36,17 +38,22 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId { private TenantId tenantId; + @NoXss private String name; + @NoXss private String description; private boolean isDefault; private DeviceProfileType type; private DeviceTransportType transportType; private DeviceProfileProvisionType provisionType; private RuleChainId defaultRuleChainId; + @NoXss private String defaultQueueName; + @Valid private transient DeviceProfileData profileData; @JsonIgnore private byte[] profileDataBytes; + @NoXss private String provisionDeviceKey; public DeviceProfile() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java index 55ec23a9b3..e8b48ee23a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.objects.TelemetryEntityView; +import org.thingsboard.server.common.data.validation.NoXss; /** * Created by Victor Basanets on 8/27/2017. @@ -39,7 +40,9 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo private EntityId entityId; private TenantId tenantId; private CustomerId customerId; + @NoXss private String name; + @NoXss private String type; private TelemetryEntityView keys; private long startTimeMs; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java index 41dc37ec72..b6adf6cf65 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Tenant.java @@ -20,13 +20,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.validation.NoXss; @EqualsAndHashCode(callSuper = true) public class Tenant extends ContactBased implements HasTenantId { private static final long serialVersionUID = 8057243243859922101L; - + + @NoXss private String title; + @NoXss private String region; private TenantProfileId tenantProfileId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java index 3796ec856d..a0fefea6cc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java @@ -23,6 +23,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.data.validation.NoXss; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -34,7 +35,9 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn @Slf4j public class TenantProfile extends SearchTextBased implements HasName { + @NoXss private String name; + @NoXss private String description; private boolean isDefault; private boolean isolatedTbCore; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/User.java b/common/data/src/main/java/org/thingsboard/server/common/data/User.java index 5792d23887..420aff71ce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/User.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/User.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; -import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.server.common.data.validation.NoXss; @EqualsAndHashCode(callSuper = true) public class User extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { @@ -35,7 +35,9 @@ public class User extends SearchTextBasedWithAdditionalInfo implements H private CustomerId customerId; private String email; private Authority authority; + @NoXss private String firstName; + @NoXss private String lastName; public User() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index 6209a11a49..f9d64cb712 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -15,12 +15,15 @@ */ package org.thingsboard.server.common.data.asset; -import com.fasterxml.jackson.databind.JsonNode; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.HasCustomerId; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.NoXss; @EqualsAndHashCode(callSuper = true) public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { @@ -29,8 +32,11 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements private TenantId tenantId; private CustomerId customerId; + @NoXss private String name; + @NoXss private String type; + @NoXss private String label; public Asset() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java index bbe20bdcaa..dee683d242 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmCondition.java @@ -17,15 +17,15 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import org.thingsboard.server.common.data.query.KeyFilter; +import javax.validation.Valid; import java.util.List; -import java.util.concurrent.TimeUnit; @Data @JsonIgnoreProperties(ignoreUnknown = true) public class AlarmCondition { + @Valid private List condition; private AlarmConditionSpec spec; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java index 86aafc19e5..b6dc9da8ba 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilter.java @@ -18,13 +18,19 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.query.EntityKeyValueType; import org.thingsboard.server.common.data.query.KeyFilterPredicate; +import org.thingsboard.server.common.data.validation.NoXss; + +import javax.validation.Valid; @Data public class AlarmConditionFilter { + @Valid private AlarmConditionFilterKey key; private EntityKeyValueType valueType; + @NoXss private Object value; + @Valid private KeyFilterPredicate predicate; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java index 33ee0b0628..a82d2e362f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmConditionFilterKey.java @@ -16,11 +16,13 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import org.thingsboard.server.common.data.validation.NoXss; @Data public class AlarmConditionFilterKey { private final AlarmConditionKeyType type; + @NoXss private final String key; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java index f04c75d75f..208cd03347 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/AlarmRule.java @@ -16,13 +16,18 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import org.thingsboard.server.common.data.validation.NoXss; + +import javax.validation.Valid; @Data public class AlarmRule { + @Valid private AlarmCondition condition; private AlarmSchedule schedule; // Advanced + @NoXss private String alarmDetails; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java index 99dc5c8cd4..ee9bdfb30f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileAlarm.java @@ -17,7 +17,9 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.validation.NoXss; +import javax.validation.Valid; import java.util.List; import java.util.TreeMap; @@ -25,9 +27,12 @@ import java.util.TreeMap; public class DeviceProfileAlarm { private String id; + @NoXss private String alarmType; + @Valid private TreeMap createRules; + @Valid private AlarmRule clearRule; // Hidden in advanced settings diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java index b709a691b9..1bf8a88a3a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileData.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; +import javax.validation.Valid; import java.util.List; @Data @@ -25,6 +26,7 @@ public class DeviceProfileData { private DeviceProfileConfiguration configuration; private DeviceProfileTransportConfiguration transportConfiguration; private DeviceProfileProvisionConfiguration provisionConfiguration; + @Valid private List alarms; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java index 2fa0a4d36a..6f824155de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/DynamicValue.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.query; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.validation.NoXss; @Data @RequiredArgsConstructor @@ -27,6 +28,7 @@ public class DynamicValue { private T resolvedValue; private final DynamicValueSourceType sourceType; + @NoXss private final String sourceAttribute; private final boolean inherit; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/FilterPredicateValue.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/FilterPredicateValue.java index 8bb227ab66..9228718b83 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/FilterPredicateValue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/FilterPredicateValue.java @@ -20,15 +20,21 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.Getter; +import org.thingsboard.server.common.data.validation.NoXss; + +import javax.validation.Valid; @Data public class FilterPredicateValue { @Getter + @NoXss private final T defaultValue; @Getter + @NoXss private final T userValue; @Getter + @Valid private final DynamicValue dynamicValue; public FilterPredicateValue(T defaultValue) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java index fffe38cd57..d3a09813e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/StringFilterPredicate.java @@ -17,10 +17,13 @@ package org.thingsboard.server.common.data.query; import lombok.Data; +import javax.validation.Valid; + @Data public class StringFilterPredicate implements SimpleKeyFilterPredicate { private StringOperation operation; + @Valid private FilterPredicateValue value; private boolean ignoreCase; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 330b7de82d..f2610696f6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.NoXss; @Data @EqualsAndHashCode(callSuper = true) @@ -35,6 +36,7 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im private static final long serialVersionUID = -5656679015121935465L; private TenantId tenantId; + @NoXss private String name; private RuleNodeId firstRuleNodeId; private boolean root; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/validation/NoXss.java b/common/data/src/main/java/org/thingsboard/server/common/data/validation/NoXss.java new file mode 100644 index 0000000000..2ecc737fee --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/validation/NoXss.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.validation; + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@Constraint(validatedBy = {}) +public @interface NoXss { + String message() default "field value is malformed"; + + Class[] groups() default {}; + + Class[] payload() default {}; +} diff --git a/dao/pom.xml b/dao/pom.xml index 5b7f0f0954..e8bd09bc66 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -107,6 +107,14 @@ com.fasterxml.jackson.core jackson-databind + + org.hibernate.validator + hibernate-validator + + + org.glassfish + javax.el + org.springframework spring-context @@ -194,6 +202,11 @@ hsqldb test + + org.junit.jupiter + junit-jupiter-params + test + org.springframework spring-context-support diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index e8b8eaf009..ff8e79efc2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -17,29 +17,50 @@ package org.thingsboard.server.dao.service; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; +import org.hibernate.validator.HibernateValidator; +import org.hibernate.validator.HibernateValidatorConfiguration; +import org.hibernate.validator.cfg.ConstraintMapping; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.validation.NoXss; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.exception.DataValidationException; +import javax.validation.ConstraintViolation; +import javax.validation.Validation; +import javax.validation.Validator; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Set; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; @Slf4j public abstract class DataValidator> { private static final Pattern EMAIL_PATTERN = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$", Pattern.CASE_INSENSITIVE); + private static Validator fieldsValidator; + + static { + initializeFieldsValidator(); + } + public void validate(D data, Function tenantIdFunction) { try { if (data == null) { throw new DataValidationException("Data object can't be null!"); } + + List validationErrors = validateFields(data); + if (!validationErrors.isEmpty()) { + throw new IllegalArgumentException("Validation error: " + String.join(", ", validationErrors)); + } + TenantId tenantId = tenantIdFunction.apply(data); validateDataImpl(tenantId, data); if (data.getId() == null) { @@ -81,6 +102,14 @@ public abstract class DataValidator> { return emailMatcher.matches(); } + private List validateFields(D data) { + Set> constraintsViolations = fieldsValidator.validate(data); + return constraintsViolations.stream() + .map(ConstraintViolation::getMessage) + .distinct() + .collect(Collectors.toList()); + } + protected void validateNumberOfEntitiesPerTenant(TenantId tenantId, TenantEntityDao tenantEntityDao, long maxEntities, @@ -111,4 +140,13 @@ public abstract class DataValidator> { throw new DataValidationException("Provided json structure is different from stored one '" + actualNode + "'!"); } } + + private static void initializeFieldsValidator() { + HibernateValidatorConfiguration validatorConfiguration = Validation.byProvider(HibernateValidator.class).configure(); + ConstraintMapping constraintMapping = validatorConfiguration.createConstraintMapping(); + constraintMapping.constraintDefinition(NoXss.class).validatedBy(NoXssValidator.class); + validatorConfiguration.addMapping(constraintMapping); + + fieldsValidator = validatorConfiguration.buildValidatorFactory().getValidator(); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java new file mode 100644 index 0000000000..e16aebbfea --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import com.google.common.io.Resources; +import lombok.extern.slf4j.Slf4j; +import org.owasp.validator.html.AntiSamy; +import org.owasp.validator.html.Policy; +import org.owasp.validator.html.PolicyException; +import org.owasp.validator.html.ScanException; +import org.thingsboard.server.common.data.validation.NoXss; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; + +@Slf4j +public class NoXssValidator implements ConstraintValidator { + private static final AntiSamy xssChecker = new AntiSamy(); + private static Policy xssPolicy; + + @Override + public void initialize(NoXss constraintAnnotation) { + if (xssPolicy == null) { + try { + xssPolicy = Policy.getInstance(Resources.getResource("xss-policy.xml")); + } catch (Exception e) { + log.error("Failed to set xss policy: {}", e.getMessage()); + } + } + } + + @Override + public boolean isValid(Object value, ConstraintValidatorContext constraintValidatorContext) { + if (!(value instanceof String) || ((String) value).isEmpty() || xssPolicy == null) { + return true; + } + + try { + return xssChecker.scan((String) value, xssPolicy).getNumberOfErrors() == 0; + } catch (ScanException | PolicyException e) { + return false; + } + } +} diff --git a/dao/src/main/resources/xss-policy.xml b/dao/src/main/resources/xss-policy.xml new file mode 100644 index 0000000000..6ea6660d2b --- /dev/null +++ b/dao/src/main/resources/xss-policy.xml @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + g + grin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java new file mode 100644 index 0000000000..8463e722cb --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import javax.validation.ConstraintValidatorContext; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.mockito.Mockito.mock; + +public class NoXssValidatorTest { + private static NoXssValidator validator; + + @BeforeAll + public static void beforeAll() { + validator = new NoXssValidator(); + validator.initialize(null); + } + + @ParameterizedTest + @ValueSource(strings = { + "aboba666", + "909090909", + "qwertyyyy", + "bambam", + "

Link!!!

1221", + "

Please log in to proceed

Username:

Password:



", + " ", + "123 bebe", + }) + public void testIsNotValid(String stringWithXss) { + boolean isValid = validator.isValid(stringWithXss, mock(ConstraintValidatorContext.class)); + assertFalse(isValid); + } + +} diff --git a/dao/src/test/resources/xss-policy.xml b/dao/src/test/resources/xss-policy.xml new file mode 100644 index 0000000000..6ea6660d2b --- /dev/null +++ b/dao/src/test/resources/xss-policy.xml @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + g + grin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index d9af0abf12..5530bdf179 100755 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ 0.7.0 2.2.0 4.12 + 5.7.1 1.7.7 1.2.3 3.3.3 @@ -113,6 +114,10 @@ 1.0.2TB 3.4.0 7.54.2 + 6.0.13.Final + 3.0.0 + 2.0.1.Final + 1.6.2 @@ -1261,6 +1266,12 @@ ${junit.version} test + + org.junit.jupiter + junit-jupiter-params + ${jupiter.version} + test + org.dbunit dbunit @@ -1458,6 +1469,36 @@ + + org.hibernate.validator + hibernate-validator + ${hibernate-validator.version} + + + org.glassfish + javax.el + ${javax.el.version} + + + javax.validation + validation-api + ${javax.validation-api.version} + + + org.owasp.antisamy + antisamy + ${antisamy.version} + + + org.slf4j + * + + + com.github.spotbugs + spotbugs-annotations + + + From 2344d6d62f4fb96b5a9904630e047c40be1a6e21 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Fri, 2 Apr 2021 12:25:05 +0300 Subject: [PATCH 21/23] Fixed RPC string params in protobuf payload type. Reverted changes from the previous fix (#4356) * Revert "added a fix for string value RPC params: removed redundant escaped characters" This reverts commit 1891af54 * removed redundant escape characters for string RPC params in Proto payload * cleanup code --- .../server/controller/RpcController.java | 10 +++----- .../mqtt/adaptors/ProtoMqttAdaptor.java | 2 +- .../transport/adaptor/JsonConverter.java | 11 +------- .../transport/adaptor/ProtoConverter.java | 25 +++++++++++++++++++ 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/RpcController.java b/application/src/main/java/org/thingsboard/server/controller/RpcController.java index f98bf364df..269755bd3e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RpcController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RpcController.java @@ -97,13 +97,9 @@ public class RpcController extends BaseController { private DeferredResult handleDeviceRPCRequest(boolean oneWay, DeviceId deviceId, String requestBody) throws ThingsboardException { try { JsonNode rpcRequestBody = jsonMapper.readTree(requestBody); - String requestData; - if (rpcRequestBody.get("params").isTextual()) { - requestData = rpcRequestBody.get("params").asText(); - } else { - requestData = jsonMapper.writeValueAsString(rpcRequestBody.get("params")); - } - RpcRequest cmd = new RpcRequest(rpcRequestBody.get("method").asText(), requestData); + RpcRequest cmd = new RpcRequest(rpcRequestBody.get("method").asText(), + jsonMapper.writeValueAsString(rpcRequestBody.get("params"))); + if (rpcRequestBody.has("timeout")) { cmd.setTimeout(rpcRequestBody.get("timeout").asLong()); } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java index a004fbd500..c948e755c2 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java @@ -146,7 +146,7 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor { @Override public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, TransportProtos.ToDeviceRpcRequestMsg rpcRequest) { - return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_RPC_REQUESTS_TOPIC + rpcRequest.getRequestId(), rpcRequest.toByteArray())); + return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_RPC_REQUESTS_TOPIC + rpcRequest.getRequestId(), ProtoConverter.convertToRpcRequest(rpcRequest))); } @Override diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index af27f4de1e..772faedc7d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -22,7 +22,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import com.google.gson.JsonSyntaxException; -import com.google.gson.stream.MalformedJsonException; import org.apache.commons.lang3.math.NumberUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.DataConstants; @@ -158,15 +157,7 @@ public class JsonConverter { result.addProperty("id", msg.getRequestId()); } result.addProperty("method", msg.getMethodName()); - try { - result.add("params", JSON_PARSER.parse(msg.getParams())); - } catch (JsonSyntaxException ex) { - if (ex.getCause() instanceof MalformedJsonException) { - result.addProperty("params", msg.getParams()); - } else { - throw ex; - } - } + result.add("params", JSON_PARSER.parse(msg.getParams())); return result; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java index fc82f78633..ed440ebcfe 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/ProtoConverter.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.common.transport.adaptor; +import com.google.gson.JsonElement; import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; @@ -167,4 +169,27 @@ public class ProtoConverter { }); return kvList; } + + public static byte[] convertToRpcRequest(TransportProtos.ToDeviceRpcRequestMsg toDeviceRpcRequestMsg) { + TransportProtos.ToDeviceRpcRequestMsg.Builder toDeviceRpcRequestMsgBuilder = toDeviceRpcRequestMsg.newBuilderForType(); + toDeviceRpcRequestMsgBuilder.mergeFrom(toDeviceRpcRequestMsg); + toDeviceRpcRequestMsgBuilder.setParams(parseParams(toDeviceRpcRequestMsg)); + TransportProtos.ToDeviceRpcRequestMsg result = toDeviceRpcRequestMsgBuilder.build(); + return result.toByteArray(); + } + + private static String parseParams(TransportProtos.ToDeviceRpcRequestMsg toDeviceRpcRequestMsg) { + String params = toDeviceRpcRequestMsg.getParams(); + JsonElement jsonElementParams = JSON_PARSER.parse(params); + if (!jsonElementParams.isJsonPrimitive()) { + return params; + } else { + JsonPrimitive primitiveParams = jsonElementParams.getAsJsonPrimitive(); + if (jsonElementParams.getAsJsonPrimitive().isString()) { + return primitiveParams.getAsString(); + } else { + return params; + } + } + } } From a98499354b3401a2eeb2282283301958dda32c05 Mon Sep 17 00:00:00 2001 From: vzikratyi-tb <65224127+vzikratyi-tb@users.noreply.github.com> Date: Mon, 5 Apr 2021 12:43:53 +0300 Subject: [PATCH 22/23] Used thingsboard/haproxy-certbot image (#4354) * Used thingsboard/haproxy-certbot image * Used 1.3.0 version of thingsboard/haproxy-certbot --- docker/docker-compose.yml | 2 +- docker/haproxy/config/haproxy.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a051a98742..67eb3eada9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -216,7 +216,7 @@ services: haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" - image: xalauc/haproxy-certbot:1.7.9 + image: thingsboard/haproxy-certbot:1.3.0 volumes: - ./haproxy/config:/config - ./haproxy/letsencrypt:/etc/letsencrypt diff --git a/docker/haproxy/config/haproxy.cfg b/docker/haproxy/config/haproxy.cfg index 5ff76cfdcd..2a457a0696 100644 --- a/docker/haproxy/config/haproxy.cfg +++ b/docker/haproxy/config/haproxy.cfg @@ -54,7 +54,7 @@ frontend http-in option forwardfor - reqadd X-Forwarded-Proto:\ http + http-request add-header "X-Forwarded-Proto" "http" acl transport_http_acl path_beg /api/v1/ acl letsencrypt_http_acl path_beg /.well-known/acme-challenge/ @@ -73,7 +73,7 @@ frontend https_in option forwardfor - reqadd X-Forwarded-Proto:\ https + http-request add-header "X-Forwarded-Proto" "https" acl transport_http_acl path_beg /api/v1/ acl tb_api_acl path_beg /api/ /swagger /webjars /v2/ /static/rulenode/ /oauth2/ /login/oauth2/ /static/widgets/ From 4a63660e1df3007abec2ccd84ed64f58e10f2c67 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 9 Apr 2021 12:37:08 +0300 Subject: [PATCH 23/23] Version set to 3.3.0-SNAPSHOT --- application/pom.xml | 2 +- common/actor/pom.xml | 2 +- common/dao-api/pom.xml | 2 +- common/data/pom.xml | 2 +- common/message/pom.xml | 2 +- common/pom.xml | 2 +- common/queue/pom.xml | 2 +- common/stats/pom.xml | 2 +- common/transport/coap/pom.xml | 2 +- common/transport/http/pom.xml | 2 +- common/transport/mqtt/pom.xml | 2 +- common/transport/pom.xml | 2 +- common/transport/transport-api/pom.xml | 2 +- common/util/pom.xml | 2 +- dao/pom.xml | 2 +- msa/black-box-tests/pom.xml | 2 +- msa/js-executor/pom.xml | 2 +- msa/pom.xml | 2 +- msa/tb-node/pom.xml | 2 +- msa/tb/pom.xml | 2 +- msa/transport/coap/pom.xml | 2 +- msa/transport/http/pom.xml | 2 +- msa/transport/mqtt/pom.xml | 2 +- msa/transport/pom.xml | 2 +- msa/web-ui/pom.xml | 2 +- netty-mqtt/pom.xml | 4 ++-- pom.xml | 2 +- rest-client/pom.xml | 2 +- rule-engine/pom.xml | 2 +- rule-engine/rule-engine-api/pom.xml | 2 +- rule-engine/rule-engine-components/pom.xml | 2 +- tools/pom.xml | 2 +- transport/coap/pom.xml | 2 +- transport/http/pom.xml | 2 +- transport/mqtt/pom.xml | 2 +- transport/pom.xml | 2 +- ui-ngx/pom.xml | 2 +- 37 files changed, 38 insertions(+), 38 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index adc3c9b22e..bbdf3a0bcd 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard application diff --git a/common/actor/pom.xml b/common/actor/pom.xml index 359a2362f9..1f4849cb79 100644 --- a/common/actor/pom.xml +++ b/common/actor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index 3d9699061e..ba30e5bb43 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/data/pom.xml b/common/data/pom.xml index 3067eba918..d011fcc5a7 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/message/pom.xml b/common/message/pom.xml index 114b48a65a..06ba4aff3b 100644 --- a/common/message/pom.xml +++ b/common/message/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/pom.xml b/common/pom.xml index 69368ffad4..11b1e02477 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard common diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 54098babcc..e0b30dd654 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/stats/pom.xml b/common/stats/pom.xml index 94cde12d3a..005c210b7b 100644 --- a/common/stats/pom.xml +++ b/common/stats/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index bd2f027e3e..bd4ea45d8b 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/http/pom.xml b/common/transport/http/pom.xml index c04ff34adb..f229627480 100644 --- a/common/transport/http/pom.xml +++ b/common/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/mqtt/pom.xml b/common/transport/mqtt/pom.xml index 4368aad588..676593804e 100644 --- a/common/transport/mqtt/pom.xml +++ b/common/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/pom.xml b/common/transport/pom.xml index 7c1c224f49..667a257a0b 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 2430cc3d23..60700bc3e7 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/util/pom.xml b/common/util/pom.xml index 2172563485..13afe4663a 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT common org.thingsboard.common diff --git a/dao/pom.xml b/dao/pom.xml index e8bd09bc66..f32e8b99d6 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard dao diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 5189242663..68331569ab 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index 769cbf6d36..a254b1acde 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/pom.xml b/msa/pom.xml index f15738fb74..3d7bf3f347 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard msa diff --git a/msa/tb-node/pom.xml b/msa/tb-node/pom.xml index df1988c3a0..133b81c79d 100644 --- a/msa/tb-node/pom.xml +++ b/msa/tb-node/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index d35762163c..efb9a709e9 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/transport/coap/pom.xml b/msa/transport/coap/pom.xml index 890e8b91e8..1285cca0ce 100644 --- a/msa/transport/coap/pom.xml +++ b/msa/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/http/pom.xml b/msa/transport/http/pom.xml index cc85b507a9..da0688b545 100644 --- a/msa/transport/http/pom.xml +++ b/msa/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/mqtt/pom.xml b/msa/transport/mqtt/pom.xml index c7352d9821..782b6a228a 100644 --- a/msa/transport/mqtt/pom.xml +++ b/msa/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index 79103f31dd..4e32be415a 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index 93c7f24e21..85f5db6ce3 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT msa org.thingsboard.msa diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index ce0963d605..3cf567a26f 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -19,11 +19,11 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard netty-mqtt - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT jar Netty MQTT Client diff --git a/pom.xml b/pom.xml index 5530bdf179..acf5c110c9 100755 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT pom Thingsboard diff --git a/rest-client/pom.xml b/rest-client/pom.xml index b8373b351f..e45db432d8 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard rest-client diff --git a/rule-engine/pom.xml b/rule-engine/pom.xml index 3709498312..d4af78f871 100644 --- a/rule-engine/pom.xml +++ b/rule-engine/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard rule-engine diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index fe44acac1f..bb348793c1 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index f435277fcc..31c0c1d824 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/tools/pom.xml b/tools/pom.xml index 4877914b62..3c1294143b 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard tools diff --git a/transport/coap/pom.xml b/transport/coap/pom.xml index c979a2d2e1..1ede01a491 100644 --- a/transport/coap/pom.xml +++ b/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/http/pom.xml b/transport/http/pom.xml index b91c5d9dd0..41ca5683a1 100644 --- a/transport/http/pom.xml +++ b/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/mqtt/pom.xml b/transport/mqtt/pom.xml index b9c651d9c6..28aaeeab39 100644 --- a/transport/mqtt/pom.xml +++ b/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/pom.xml b/transport/pom.xml index b3ebf1fbd0..81a5ce8116 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard transport diff --git a/ui-ngx/pom.xml b/ui-ngx/pom.xml index f06442db25..fb8c809260 100644 --- a/ui-ngx/pom.xml +++ b/ui-ngx/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 3.2.2-SNAPSHOT + 3.3.0-SNAPSHOT thingsboard org.thingsboard