Browse Source

Merge remote-tracking branch 'upstream/master' into feature/dynamic-oauth2-new-table

# Conflicts:
#	application/src/main/data/upgrade/3.1.0/schema_update.sql
pull/3557/head
vzikratyi 6 years ago
parent
commit
faf0184cff
  1. 2
      application/pom.xml
  2. 14
      application/src/main/data/json/system/widget_bundles/input_widgets.json
  3. 35
      application/src/main/data/upgrade/3.0.1/schema_ts_latest.sql
  4. 4
      application/src/main/data/upgrade/3.1.0/schema_update.sql
  5. 8
      application/src/main/java/org/thingsboard/server/controller/AdminController.java
  6. 175
      application/src/main/java/org/thingsboard/server/controller/EntityViewController.java
  7. 4
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  8. 8
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  9. 46
      application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java
  10. 102
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  11. 39
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  12. 18
      application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java
  13. 14
      application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java
  14. 35
      application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java
  15. 20
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java
  16. 30
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java
  17. 7
      application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java
  18. 18
      application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java
  19. 96
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  20. 7
      application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataUpdate.java
  21. 20
      application/src/main/resources/thingsboard.yml
  22. 2
      common/actor/pom.xml
  23. 2
      common/dao-api/pom.xml
  24. 6
      common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java
  25. 2
      common/data/pom.xml
  26. 2
      common/message/pom.xml
  27. 2
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  28. 2
      common/pom.xml
  29. 2
      common/queue/pom.xml
  30. 42
      common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java
  31. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java
  32. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java
  33. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java
  34. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java
  35. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java
  36. 2
      common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java
  37. 13
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java
  38. 16
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java
  39. 16
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java
  40. 3
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java
  41. 2
      common/stats/pom.xml
  42. 37
      common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java
  43. 4
      common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java
  44. 2
      common/transport/coap/pom.xml
  45. 2
      common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java
  46. 2
      common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java
  47. 2
      common/transport/http/pom.xml
  48. 2
      common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
  49. 2
      common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java
  50. 2
      common/transport/mqtt/pom.xml
  51. 2
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java
  52. 2
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java
  53. 2
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java
  54. 1
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java
  55. 2
      common/transport/pom.xml
  56. 2
      common/transport/transport-api/pom.xml
  57. 2
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java
  58. 2
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  59. 2
      common/util/pom.xml
  60. 2
      dao/pom.xml
  61. 8
      dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java
  62. 8
      dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java
  63. 30
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java
  64. 88
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java
  65. 40
      dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java
  66. 7
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java
  67. 29
      dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java
  68. 21
      dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java
  69. 76
      dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java
  70. 14
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  71. 11
      dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java
  72. 2
      dao/src/main/resources/sql/schema-entities-idx.sql
  73. 12
      dao/src/main/resources/sql/schema-entities.sql
  74. 14
      dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java
  75. 10
      dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java
  76. 40
      dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java
  77. 20
      dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java
  78. 5
      docker/compose-utils.sh
  79. 57
      docker/docker-compose.confluent.yml
  80. 18
      docker/queue-confluent.env
  81. 2
      msa/black-box-tests/pom.xml
  82. 39
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java
  83. 376
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java
  84. 6
      msa/js-executor/config/custom-environment-variables.yml
  85. 4
      msa/js-executor/config/default.yml
  86. 1
      msa/js-executor/docker/Dockerfile
  87. 2
      msa/js-executor/package-lock.json
  88. 2
      msa/js-executor/package.json
  89. 2
      msa/js-executor/pom.xml
  90. 20
      msa/js-executor/queue/kafkaTemplate.js
  91. 2
      msa/pom.xml
  92. 1
      msa/tb-node/docker/Dockerfile
  93. 2
      msa/tb-node/pom.xml
  94. 1
      msa/tb/docker-cassandra/Dockerfile
  95. 1
      msa/tb/docker-postgres/Dockerfile
  96. 1
      msa/tb/docker-tb/Dockerfile
  97. 2
      msa/tb/pom.xml
  98. 1
      msa/transport/coap/docker/Dockerfile
  99. 2
      msa/transport/coap/pom.xml
  100. 1
      msa/transport/http/docker/Dockerfile

2
application/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>application</artifactId>

14
application/src/main/data/json/system/widget_bundles/input_widgets.json

File diff suppressed because one or more lines are too long

35
application/src/main/data/upgrade/3.0.1/schema_ts_latest.sql

@ -0,0 +1,35 @@
--
-- Copyright © 2016-2020 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE TABLE IF NOT EXISTS ts_kv_latest
(
entity_id uuid NOT NULL,
key int NOT NULL,
ts bigint NOT NULL,
bool_v boolean,
str_v varchar(10000000),
long_v bigint,
dbl_v double precision,
json_v json,
CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key)
);
CREATE TABLE IF NOT EXISTS ts_kv_dictionary
(
key varchar(255) NOT NULL,
key_id serial UNIQUE,
CONSTRAINT ts_key_id_pkey PRIMARY KEY (key)
);

4
application/src/main/data/upgrade/3.1.0/schema_update.sql

@ -78,4 +78,6 @@ CREATE TABLE IF NOT EXISTS oauth2_client_registration_template (
icon varchar(255),
help_link varchar(255),
CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id)
);
);
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC);

8
application/src/main/java/org/thingsboard/server/controller/AdminController.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.controller;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
@ -59,7 +60,11 @@ public class AdminController extends BaseController {
public AdminSettings getAdminSettings(@PathVariable("key") String key) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ);
return checkNotNull(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, key));
AdminSettings adminSettings = checkNotNull(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, key));
if (adminSettings.getKey().equals("mail")) {
((ObjectNode) adminSettings.getJsonValue()).put("password", "");
}
return adminSettings;
} catch (Exception e) {
throw handleException(e);
}
@ -74,6 +79,7 @@ public class AdminController extends BaseController {
adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings));
if (adminSettings.getKey().equals("mail")) {
mailService.updateMailConfiguration();
((ObjectNode) adminSettings.getJsonValue()).put("password", "");
}
return adminSettings;
} catch (Exception e) {

175
application/src/main/java/org/thingsboard/server/controller/EntityViewController.java

@ -19,7 +19,9 @@ import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
@ -40,10 +42,14 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
@ -52,10 +58,12 @@ import org.thingsboard.server.service.security.permission.Resource;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.thingsboard.server.controller.CustomerController.CUSTOMER_ID;
/**
@ -69,6 +77,9 @@ public class EntityViewController extends BaseController {
public static final String ENTITY_VIEW_ID = "entityViewId";
@Autowired
private TimeseriesService tsService;
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/entityView/{entityViewId}", method = RequestMethod.GET)
@ResponseBody
@ -101,16 +112,35 @@ public class EntityViewController extends BaseController {
try {
entityView.setTenantId(getCurrentUser().getTenantId());
checkEntity(entityView.getId(), entityView, Resource.ENTITY_VIEW);
List<ListenableFuture<?>> futures = new ArrayList<>();
if (entityView.getId() == null) {
accessControlService
.checkPermission(getCurrentUser(), Resource.ENTITY_VIEW, Operation.CREATE, null, entityView);
} else {
EntityView existingEntityView = checkEntityViewId(entityView.getId(), Operation.WRITE);
if (existingEntityView.getKeys() != null) {
if (existingEntityView.getKeys().getAttributes() != null) {
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.CLIENT_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), getCurrentUser()));
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SERVER_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), getCurrentUser()));
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SHARED_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), getCurrentUser()));
}
}
List<String> tsKeys = existingEntityView.getKeys() != null && existingEntityView.getKeys().getTimeseries() != null ?
existingEntityView.getKeys().getTimeseries() : Collections.emptyList();
futures.add(deleteLatestFromEntityView(existingEntityView, tsKeys, getCurrentUser()));
}
EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView));
List<ListenableFuture<List<Void>>> futures = new ArrayList<>();
if (savedEntityView.getKeys() != null && savedEntityView.getKeys().getAttributes() != null) {
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), getCurrentUser()));
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SERVER_SCOPE, savedEntityView.getKeys().getAttributes().getSs(), getCurrentUser()));
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SHARED_SCOPE, savedEntityView.getKeys().getAttributes().getSh(), getCurrentUser()));
if (savedEntityView.getKeys() != null) {
if (savedEntityView.getKeys().getAttributes() != null) {
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), getCurrentUser()));
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SERVER_SCOPE, savedEntityView.getKeys().getAttributes().getSs(), getCurrentUser()));
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.SHARED_SCOPE, savedEntityView.getKeys().getAttributes().getSh(), getCurrentUser()));
}
futures.add(copyLatestFromEntityToEntityView(savedEntityView, getCurrentUser()));
}
for (ListenableFuture<List<Void>> future : futures) {
for (ListenableFuture<?> future : futures) {
try {
future.get();
} catch (InterruptedException | ExecutionException e) {
@ -128,6 +158,125 @@ public class EntityViewController extends BaseController {
}
}
private ListenableFuture<Void> deleteLatestFromEntityView(EntityView entityView, List<String> keys, SecurityUser user) {
EntityViewId entityId = entityView.getId();
SettableFuture<Void> resultFuture = SettableFuture.create();
if (keys != null && !keys.isEmpty()) {
tsSubService.deleteLatest(entityView.getTenantId(), entityId, keys, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void tmp) {
try {
logTimeseriesDeleted(user, entityId, keys, null);
} catch (ThingsboardException e) {
log.error("Failed to log timeseries delete", e);
}
resultFuture.set(tmp);
}
@Override
public void onFailure(Throwable t) {
try {
logTimeseriesDeleted(user, entityId, keys, t);
} catch (ThingsboardException e) {
log.error("Failed to log timeseries delete", e);
}
resultFuture.setException(t);
}
});
} else {
tsSubService.deleteAllLatest(entityView.getTenantId(), entityId, new FutureCallback<Collection<String>>() {
@Override
public void onSuccess(@Nullable Collection<String> keys) {
try {
logTimeseriesDeleted(user, entityId, new ArrayList<>(keys), null);
} catch (ThingsboardException e) {
log.error("Failed to log timeseries delete", e);
}
resultFuture.set(null);
}
@Override
public void onFailure(Throwable t) {
try {
logTimeseriesDeleted(user, entityId, Collections.emptyList(), t);
} catch (ThingsboardException e) {
log.error("Failed to log timeseries delete", e);
}
resultFuture.setException(t);
}
});
}
return resultFuture;
}
private ListenableFuture<Void> deleteAttributesFromEntityView(EntityView entityView, String scope, List<String> keys, SecurityUser user) {
EntityViewId entityId = entityView.getId();
SettableFuture<Void> resultFuture = SettableFuture.create();
if (keys != null && !keys.isEmpty()) {
tsSubService.deleteAndNotify(entityView.getTenantId(), entityId, scope, keys, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void tmp) {
try {
logAttributesDeleted(user, entityId, scope, keys, null);
} catch (ThingsboardException e) {
log.error("Failed to log attribute delete", e);
}
resultFuture.set(tmp);
}
@Override
public void onFailure(Throwable t) {
try {
logAttributesDeleted(user, entityId, scope, keys, t);
} catch (ThingsboardException e) {
log.error("Failed to log attribute delete", e);
}
resultFuture.setException(t);
}
});
} else {
resultFuture.set(null);
}
return resultFuture;
}
private ListenableFuture<List<Void>> copyLatestFromEntityToEntityView(EntityView entityView, SecurityUser user) {
EntityViewId entityId = entityView.getId();
List<String> keys = entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null ?
entityView.getKeys().getTimeseries() : Collections.emptyList();
long startTs = entityView.getStartTimeMs();
long endTs = entityView.getEndTimeMs() == 0 ? Long.MAX_VALUE : entityView.getEndTimeMs();
ListenableFuture<List<String>> keysFuture;
if (keys.isEmpty()) {
keysFuture = Futures.transform(tsService.findAllLatest(user.getTenantId(),
entityView.getEntityId()), latest -> latest.stream().map(TsKvEntry::getKey).collect(Collectors.toList()), MoreExecutors.directExecutor());
} else {
keysFuture = Futures.immediateFuture(keys);
}
ListenableFuture<List<TsKvEntry>> latestFuture = Futures.transformAsync(keysFuture, fetchKeys -> {
List<ReadTsKvQuery> queries = fetchKeys.stream().filter(key -> !isBlank(key)).map(key -> new BaseReadTsKvQuery(key, startTs, endTs, 1, "DESC")).collect(Collectors.toList());
if (!queries.isEmpty()) {
return tsService.findAll(user.getTenantId(), entityView.getEntityId(), queries);
} else {
return Futures.immediateFuture(null);
}
}, MoreExecutors.directExecutor());
return Futures.transform(latestFuture, latestValues -> {
if (latestValues != null && !latestValues.isEmpty()) {
tsSubService.saveLatestAndNotify(entityView.getTenantId(), entityId, latestValues, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void tmp) {
}
@Override
public void onFailure(Throwable t) {
}
});
}
return null;
}, MoreExecutors.directExecutor());
}
private ListenableFuture<List<Void>> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection<String> keys, SecurityUser user) throws ThingsboardException {
EntityViewId entityId = entityView.getId();
if (keys != null && !keys.isEmpty()) {
@ -174,10 +323,20 @@ public class EntityViewController extends BaseController {
}
private void logAttributesUpdated(SecurityUser user, EntityId entityId, String scope, List<AttributeKvEntry> attributes, Throwable e) throws ThingsboardException {
logEntityAction(user, (UUIDBased & EntityId) entityId, null, null, ActionType.ATTRIBUTES_UPDATED, toException(e),
logEntityAction(user, entityId, null, null, ActionType.ATTRIBUTES_UPDATED, toException(e),
scope, attributes);
}
private void logAttributesDeleted(SecurityUser user, EntityId entityId, String scope, List<String> keys, Throwable e) throws ThingsboardException {
logEntityAction(user, entityId, null, null, ActionType.ATTRIBUTES_DELETED, toException(e),
scope, keys);
}
private void logTimeseriesDeleted(SecurityUser user, EntityId entityId, List<String> keys, Throwable e) throws ThingsboardException {
logEntityAction(user, entityId, null, null, ActionType.TIMESERIES_DELETED, toException(e),
keys);
}
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/entityView/{entityViewId}", method = RequestMethod.DELETE)
@ResponseStatus(value = HttpStatus.OK)

4
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -171,6 +171,10 @@ public class ThingsboardInstallService {
case "3.0.1":
log.info("Upgrading ThingsBoard from version 3.0.1 to 3.1.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.0.1");
dataUpdateService.updateData("3.0.1");
case "3.1.0":
log.info("Upgrading ThingsBoard from version 3.1.0 to 3.1.1 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.1.0");
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
case "3.1.0":

8
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -304,6 +304,14 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
log.error("Failed updating schema!!!", e);
}
break;
case "3.1.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.1.0", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Schema updated.");
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

46
application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java

@ -29,9 +29,14 @@ import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryComposite
import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity;
import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository;
import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository;
import org.thingsboard.server.dao.util.NoSqlAnyDao;
import org.thingsboard.server.service.install.EntityDatabaseSchemaService;
import org.thingsboard.server.dao.util.NoSqlTsDao;
import org.thingsboard.server.dao.util.SqlTsLatestDao;
import org.thingsboard.server.service.install.InstallScripts;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Arrays;
@ -51,12 +56,13 @@ import static org.thingsboard.server.service.install.migrate.CassandraToSqlColum
@Service
@Profile("install")
@NoSqlAnyDao
@NoSqlTsDao
@SqlTsLatestDao
@Slf4j
public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService {
@Autowired
private EntityDatabaseSchemaService entityDatabaseSchemaService;
private static final int MAX_KEY_LENGTH = 255;
private static final int MAX_STR_V_LENGTH = 10000000;
@Autowired
private InsertLatestTsRepository insertLatestTsRepository;
@ -67,6 +73,9 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
@Autowired
protected TsKvDictionaryRepository dictionaryRepository;
@Autowired
private InstallScripts installScripts;
@Value("${spring.datasource.url}")
protected String dbUrl;
@ -83,8 +92,9 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
@Override
public void migrate() throws Exception {
log.info("Performing migration of latest timeseries data from cassandra to SQL database ...");
entityDatabaseSchemaService.createDatabaseSchema(false);
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.0.1", "schema_ts_latest.sql");
loadSql(schemaUpdateFile, conn);
conn.setAutoCommit(false);
for (CassandraToSqlTable table : tables) {
table.migrateToSql(cluster.getSession(), conn);
@ -93,11 +103,10 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
log.error("Unexpected error during ThingsBoard entities data migration!", e);
throw e;
}
entityDatabaseSchemaService.createDatabaseIndexes();
}
private List<CassandraToSqlTable> tables = Arrays.asList(
new CassandraToSqlTable("ts_kv_latest_cf",
new CassandraToSqlTable("ts_kv_latest_cf", "ts_kv_latest",
idColumn("entity_id"),
stringColumn("key"),
bigintColumn("ts"),
@ -127,6 +136,12 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
String strV = data[4].getValue();
if (strV != null) {
if (strV.length() > MAX_STR_V_LENGTH) {
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [str_v] and will be truncated!",
strV.length(), MAX_STR_V_LENGTH);
log.warn("Affected data:\n{}", strV);
strV = strV.substring(0, MAX_STR_V_LENGTH);
}
latestEntity.setStrValue(strV);
} else {
Long longV = null;
@ -168,6 +183,13 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
}
protected Integer getOrSaveKeyId(String strKey) {
if (strKey.length() > MAX_KEY_LENGTH) {
log.warn("[ts_kv_latest] Value size [{}] exceeds maximum size [{}] of column [key] and will be truncated!",
strKey.length(), MAX_KEY_LENGTH);
log.warn("Affected data:\n{}", strKey);
strKey = strKey.substring(0, MAX_KEY_LENGTH);
}
Integer keyId = tsKvDictionaryMap.get(strKey);
if (keyId == null) {
Optional<TsKvDictionary> tsKvDictionaryOptional;
@ -202,4 +224,10 @@ public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateServ
}
return keyId;
}
private void loadSql(Path sqlFile, Connection conn) throws Exception {
String sql = new String(Files.readAllBytes(sqlFile), Charset.forName("UTF-8"));
conn.createStatement().execute(sql); //NOSONAR, ignoring because method used to execute thingsboard database upgrade script
Thread.sleep(5000);
}
}

102
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -15,20 +15,41 @@
*/
package org.thingsboard.server.service.install.update;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.SearchTextBased;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.service.install.InstallScripts;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static org.apache.commons.lang.StringUtils.isBlank;
@Service
@Profile("install")
@Slf4j
@ -43,6 +64,12 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private InstallScripts installScripts;
@Autowired
private EntityViewService entityViewService;
@Autowired
private TimeseriesService tsService;
@Override
public void updateData(String fromVersion) throws Exception {
switch (fromVersion) {
@ -50,6 +77,10 @@ public class DefaultDataUpdateService implements DataUpdateService {
log.info("Updating data from version 1.4.0 to 2.0.0 ...");
tenantsDefaultRuleChainUpdater.updateEntities(null);
break;
case "3.0.1":
log.info("Updating data from version 3.0.1 to 3.1.0 ...");
tenantsEntityViewsUpdater.updateEntities(null);
break;
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
}
@ -76,4 +107,73 @@ public class DefaultDataUpdateService implements DataUpdateService {
}
};
}
private PaginatedUpdater<String, Tenant> tenantsEntityViewsUpdater =
new PaginatedUpdater<String, Tenant>() {
@Override
protected PageData<Tenant> findEntities(String region, PageLink pageLink) {
return tenantService.findTenants(pageLink);
}
@Override
protected void updateEntity(Tenant tenant) {
updateTenantEntityViews(tenant.getId());
}
};
private void updateTenantEntityViews(TenantId tenantId) {
PageLink pageLink = new PageLink(100);
PageData<EntityView> pageData = entityViewService.findEntityViewByTenantId(tenantId, pageLink);
boolean hasNext = true;
while (hasNext) {
List<ListenableFuture<List<Void>>> updateFutures = new ArrayList<>();
for (EntityView entityView : pageData.getData()) {
updateFutures.add(updateEntityViewLatestTelemetry(entityView));
}
try {
Futures.allAsList(updateFutures).get();
} catch (InterruptedException | ExecutionException e) {
log.error("Failed to copy latest telemetry to entity view", e);
}
if (pageData.hasNext()) {
pageLink = pageLink.nextPageLink();
pageData = entityViewService.findEntityViewByTenantId(tenantId, pageLink);
} else {
hasNext = false;
}
}
}
private ListenableFuture<List<Void>> updateEntityViewLatestTelemetry(EntityView entityView) {
EntityViewId entityId = entityView.getId();
List<String> keys = entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null ?
entityView.getKeys().getTimeseries() : Collections.emptyList();
long startTs = entityView.getStartTimeMs();
long endTs = entityView.getEndTimeMs() == 0 ? Long.MAX_VALUE : entityView.getEndTimeMs();
ListenableFuture<List<String>> keysFuture;
if (keys.isEmpty()) {
keysFuture = Futures.transform(tsService.findAllLatest(TenantId.SYS_TENANT_ID,
entityView.getEntityId()), latest -> latest.stream().map(TsKvEntry::getKey).collect(Collectors.toList()), MoreExecutors.directExecutor());
} else {
keysFuture = Futures.immediateFuture(keys);
}
ListenableFuture<List<TsKvEntry>> latestFuture = Futures.transformAsync(keysFuture, fetchKeys -> {
List<ReadTsKvQuery> queries = fetchKeys.stream().filter(key -> !isBlank(key)).map(key -> new BaseReadTsKvQuery(key, startTs, endTs, 1, "DESC")).collect(Collectors.toList());
if (!queries.isEmpty()) {
return tsService.findAll(TenantId.SYS_TENANT_ID, entityView.getEntityId(), queries);
} else {
return Futures.immediateFuture(null);
}
}, MoreExecutors.directExecutor());
return Futures.transformAsync(latestFuture, latestValues -> {
if (latestValues != null && !latestValues.isEmpty()) {
ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(TenantId.SYS_TENANT_ID, entityId, latestValues);
return saveFuture;
}
return Futures.immediateFuture(null);
}, MoreExecutors.directExecutor());
}
}

39
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -22,17 +22,11 @@ import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.RpcError;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.*;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@ -42,40 +36,25 @@ import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.service.encoding.DataDecodingEncodingService;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.service.queue.processing.*;
import org.thingsboard.server.service.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.*;
import java.util.concurrent.*;
@Service
@TbRuleEngineComponent
@Slf4j
public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<ToRuleEngineNotificationMsg> implements TbRuleEngineConsumerService {
public static final String SUCCESSFUL_STATUS = "successful";
public static final String FAILED_STATUS = "failed";
@Value("${queue.rule-engine.poll-interval}")
private long pollDuration;
@Value("${queue.rule-engine.pack-processing-timeout}")
@ -170,7 +149,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
log.trace("[{}] Creating callback for message: {}", id, msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = new TenantId(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
TbMsgCallback callback = new TbMsgPackCallback(id, tenantId, ctx);
TbMsgCallback callback = statsEnabled ?
new TbMsgPackCallback(id, tenantId, ctx, stats.getTimer(tenantId, SUCCESSFUL_STATUS), stats.getTimer(tenantId, FAILED_STATUS)) :
new TbMsgPackCallback(id, tenantId, ctx);
try {
if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback);

18
application/src/main/java/org/thingsboard/server/service/queue/TbMsgPackCallback.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.queue;
import io.micrometer.core.instrument.Timer;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
@ -23,28 +24,45 @@ import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@Slf4j
public class TbMsgPackCallback implements TbMsgCallback {
private final UUID id;
private final TenantId tenantId;
private final TbMsgPackProcessingContext ctx;
private final long startMsgProcessing;
private final Timer successfulMsgTimer;
private final Timer failedMsgTimer;
public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx) {
this(id, tenantId, ctx, null, null);
}
public TbMsgPackCallback(UUID id, TenantId tenantId, TbMsgPackProcessingContext ctx, Timer successfulMsgTimer, Timer failedMsgTimer) {
this.id = id;
this.tenantId = tenantId;
this.ctx = ctx;
this.successfulMsgTimer = successfulMsgTimer;
this.failedMsgTimer = failedMsgTimer;
startMsgProcessing = System.currentTimeMillis();
}
@Override
public void onSuccess() {
log.trace("[{}] ON SUCCESS", id);
if (successfulMsgTimer != null) {
successfulMsgTimer.record(System.currentTimeMillis() - startMsgProcessing, TimeUnit.MILLISECONDS);
}
ctx.onSuccess(id);
}
@Override
public void onFailure(RuleEngineException e) {
log.trace("[{}] ON FAILURE", id, e);
if (failedMsgTimer != null) {
failedMsgTimer.record(System.currentTimeMillis() - startMsgProcessing, TimeUnit.MILLISECONDS);
}
ctx.onFailure(tenantId, id, e);
}

14
application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.service.queue;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Timer;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
@ -41,6 +43,8 @@ public class TbRuleEngineConsumerStats {
public static final String SUCCESSFUL_ITERATIONS = "successfulIterations";
public static final String FAILED_ITERATIONS = "failedIterations";
private final StatsFactory statsFactory;
private final StatsCounter totalMsgCounter;
private final StatsCounter successMsgCounter;
private final StatsCounter tmpTimeoutMsgCounter;
@ -54,12 +58,14 @@ public class TbRuleEngineConsumerStats {
private final List<StatsCounter> counters = new ArrayList<>();
private final ConcurrentMap<UUID, TbTenantRuleEngineStats> tenantStats = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, Timer> tenantMsgProcessTimers = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, RuleEngineException> tenantExceptions = new ConcurrentHashMap<>();
private final String queueName;
public TbRuleEngineConsumerStats(String queueName, StatsFactory statsFactory) {
this.queueName = queueName;
this.statsFactory = statsFactory;
String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName;
this.totalMsgCounter = statsFactory.createStatsCounter(statsKey, TOTAL_MSGS);
@ -82,6 +88,14 @@ public class TbRuleEngineConsumerStats {
counters.add(failedIterationsCounter);
}
public Timer getTimer(TenantId tenantId, String status){
return tenantMsgProcessTimers.computeIfAbsent(tenantId,
id -> statsFactory.createTimer(StatsType.RULE_ENGINE.getName() + "." + queueName,
"tenantId", tenantId.getId().toString(),
"status", status
));
}
public void log(TbRuleEngineProcessingResult msg, boolean finalIterationForPack) {
int success = msg.getSuccessMap().size();
int pending = msg.getPendingMap().size();

35
application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java

@ -15,7 +15,7 @@
*/
package org.thingsboard.server.service.state;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Function;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
@ -35,26 +35,27 @@ import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.util.mapping.JacksonUtil;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
@ -90,7 +91,6 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE;
@Slf4j
public class DefaultDeviceStateService implements DeviceStateService {
private static final ObjectMapper json = new ObjectMapper();
public static final String ACTIVITY_STATE = "active";
public static final String LAST_CONNECT_TIME = "lastConnectTime";
public static final String LAST_DISCONNECT_TIME = "lastDisconnectTime";
@ -197,15 +197,15 @@ public class DefaultDeviceStateService implements DeviceStateService {
if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) {
DeviceStateData stateData = getOrFetchDeviceStateData(deviceId);
if (stateData != null) {
DeviceState state = stateData.getState();
stateData.getState().setLastActivityTime(lastReportedActivity);
stateData.getMetaData().putValue("scope", SERVER_SCOPE);
pushRuleEngineMessage(stateData, ACTIVITY_EVENT);
save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity);
deviceLastSavedActivity.put(deviceId, lastReportedActivity);
DeviceState state = stateData.getState();
state.setLastActivityTime(lastReportedActivity);
if (!state.isActive()) {
state.setActive(true);
save(deviceId, ACTIVITY_STATE, state.isActive());
stateData.getMetaData().putValue("scope", SERVER_SCOPE);
pushRuleEngineMessage(stateData, ACTIVITY_EVENT);
}
}
}
@ -503,8 +503,15 @@ public class DefaultDeviceStateService implements DeviceStateService {
private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) {
DeviceState state = stateData.getState();
try {
TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON
, json.writeValueAsString(state));
String data;
if (msgType.equals(CONNECT_EVENT)) {
ObjectNode stateNode = JacksonUtil.convertValue(state, ObjectNode.class);
stateNode.remove(ACTIVITY_STATE);
data = JacksonUtil.toString(stateNode);
} else {
data = JacksonUtil.toString(state);
}
TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON, data);
clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null);
} catch (Exception e) {
log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e);

20
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java

@ -128,6 +128,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
private long dynamicPageLinkRefreshInterval;
@Value("${server.ws.dynamic_page_link.refresh_pool_size:1}")
private int dynamicPageLinkRefreshPoolSize;
@Value("${server.ws.max_entities_per_data_subscription:1000}")
private int maxEntitiesPerDataSubscription;
@Value("${server.ws.max_entities_per_alarm_subscription:1000}")
private int maxEntitiesPerAlarmSubscription;
@ -220,7 +222,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
} else if (cmd.getTsCmd() != null) {
handleTimeSeriesCmd(theCtx, cmd.getTsCmd());
} else if (!theCtx.isInitialDataSent()) {
EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null);
EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription());
wsService.sendWsMsg(theCtx.getSessionId(), update);
theCtx.setInitialDataSent(true);
}
@ -298,7 +300,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
private TbEntityDataSubCtx createSubCtx(TelemetryWebSocketSessionRef sessionRef, EntityDataCmd cmd) {
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new HashMap<>());
TbEntityDataSubCtx ctx = new TbEntityDataSubCtx(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmd.getCmdId());
TbEntityDataSubCtx ctx = new TbEntityDataSubCtx(serviceId, wsService, entityService, localSubscriptionService,
attributesService, stats, sessionRef, cmd.getCmdId(), maxEntitiesPerDataSubscription);
ctx.setAndResolveQuery(cmd.getQuery());
sessionSubs.put(cmd.getCmdId(), ctx);
return ctx;
@ -306,7 +309,8 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
private TbAlarmDataSubCtx createSubCtx(TelemetryWebSocketSessionRef sessionRef, AlarmDataCmd cmd) {
Map<Integer, TbAbstractDataSubCtx> sessionSubs = subscriptionsBySessionId.computeIfAbsent(sessionRef.getSessionId(), k -> new HashMap<>());
TbAlarmDataSubCtx ctx = new TbAlarmDataSubCtx(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription);
TbAlarmDataSubCtx ctx = new TbAlarmDataSubCtx(serviceId, wsService, entityService, localSubscriptionService,
attributesService, stats, alarmService, sessionRef, cmd.getCmdId(), maxEntitiesPerAlarmSubscription);
ctx.setAndResolveQuery(cmd.getQuery());
sessionSubs.put(cmd.getCmdId(), ctx);
return ctx;
@ -372,10 +376,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
});
EntityDataUpdate update;
if (!ctx.isInitialDataSent()) {
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
ctx.setInitialDataSent(true);
} else {
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData());
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription());
}
wsService.sendWsMsg(ctx.getSessionId(), update);
if (subscribe) {
@ -422,10 +426,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
});
EntityDataUpdate update;
if (!ctx.isInitialDataSent()) {
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
ctx.setInitialDataSent(true);
} else {
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData());
update = new EntityDataUpdate(ctx.getCmdId(), null, ctx.getData().getData(), ctx.getMaxEntitiesPerDataSubscription());
}
wsService.sendWsMsg(ctx.getSessionId(), update);
ctx.createSubscriptions(latestCmd.getKeys(), true);
@ -440,7 +444,7 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc
}, wsCallBackExecutor);
} else {
if (!ctx.isInitialDataSent()) {
EntityDataUpdate update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null);
EntityDataUpdate update = new EntityDataUpdate(ctx.getCmdId(), ctx.getData(), null, ctx.getMaxEntitiesPerDataSubscription());
wsService.sendWsMsg(ctx.getSessionId(), update);
ctx.setInitialDataSent(true);
}

30
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java

@ -36,6 +36,7 @@ import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.queue.TbClusterService;
import org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocketService;
import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate;
import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate;
@ -114,11 +115,6 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer
//TODO 3.1: replace null callbacks with callbacks from websocket service.
@Override
public void addSubscription(TbSubscription subscription) {
EntityId entityId = subscription.getEntityId();
// Telemetry subscription on Entity Views are handled differently, because we need to allow only certain keys and time ranges;
if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW) && TbSubscriptionType.TIMESERIES.equals(subscription.getType())) {
subscription = resolveEntityViewSubscription((TbTimeseriesSubscription) subscription);
}
pushSubscriptionToManagerService(subscription, true);
registerSubscription(subscription);
}
@ -203,30 +199,6 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer
}
}
private TbSubscription resolveEntityViewSubscription(TbTimeseriesSubscription subscription) {
EntityView entityView = entityViewService.findEntityViewById(TenantId.SYS_TENANT_ID, new EntityViewId(subscription.getEntityId().getId()));
Map<String, Long> keyStates;
if (subscription.isAllKeys()) {
keyStates = entityView.getKeys().getTimeseries().stream().collect(Collectors.toMap(k -> k, k -> 0L));
} else {
keyStates = subscription.getKeyStates().entrySet()
.stream().filter(entry -> entityView.getKeys().getTimeseries().contains(entry.getKey()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
return TbTimeseriesSubscription.builder()
.serviceId(subscription.getServiceId())
.sessionId(subscription.getSessionId())
.subscriptionId(subscription.getSubscriptionId())
.tenantId(subscription.getTenantId())
.entityId(entityView.getEntityId())
.startTime(entityView.getStartTimeMs())
.endTime(entityView.getEndTimeMs())
.allKeys(false)
.keyStates(keyStates).build();
}
private void registerSubscription(TbSubscription subscription) {
Map<Integer, TbSubscription> sessionSubscriptions = subscriptionsBySessionId.computeIfAbsent(subscription.getSessionId(), k -> new ConcurrentHashMap<>());
sessionSubscriptions.put(subscription.getSubscriptionId(), subscription);

7
application/src/main/java/org/thingsboard/server/service/subscription/TbAbstractDataSubCtx.java

@ -57,6 +57,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ScheduledFuture;
import java.util.function.Function;
@ -98,9 +99,9 @@ public abstract class TbAbstractDataSubCtx<T extends AbstractDataQuery<? extends
this.stats = stats;
this.sessionRef = sessionRef;
this.cmdId = cmdId;
this.subToEntityIdMap = new HashMap<>();
this.subToDynamicValueKeySet = new HashSet<>();
this.dynamicValues = new HashMap<>();
this.subToEntityIdMap = new ConcurrentHashMap<>();
this.subToDynamicValueKeySet = ConcurrentHashMap.newKeySet();
this.dynamicValues = new ConcurrentHashMap<>();
}
public void setAndResolveQuery(T query) {

18
application/src/main/java/org/thingsboard/server/service/subscription/TbEntityDataSubCtx.java

@ -15,13 +15,10 @@
*/
package org.thingsboard.server.service.subscription;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.common.data.query.EntityDataQuery;
import org.thingsboard.server.common.data.query.EntityKey;
@ -38,8 +35,6 @@ import org.thingsboard.server.service.telemetry.cmd.v2.TimeSeriesCmd;
import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@ -48,8 +43,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ScheduledFuture;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@ -63,11 +56,14 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx<EntityDataQuery> {
private boolean initialDataSent;
private TimeSeriesCmd curTsCmd;
private LatestValueCmd latestValueCmd;
@Getter
private final int maxEntitiesPerDataSubscription;
public TbEntityDataSubCtx(String serviceId, TelemetryWebSocketService wsService, EntityService entityService,
TbLocalSubscriptionService localSubscriptionService, AttributesService attributesService,
SubscriptionServiceStatistics stats, TelemetryWebSocketSessionRef sessionRef, int cmdId) {
SubscriptionServiceStatistics stats, TelemetryWebSocketSessionRef sessionRef, int cmdId, int maxEntitiesPerDataSubscription) {
super(serviceId, wsService, entityService, localSubscriptionService, attributesService, stats, sessionRef, cmdId);
this.maxEntitiesPerDataSubscription = maxEntitiesPerDataSubscription;
}
@Override
@ -120,7 +116,7 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx<EntityDataQuery> {
if (!latestUpdate.isEmpty()) {
Map<EntityKeyType, Map<String, TsValue>> latestMap = Collections.singletonMap(keyType, latestUpdate);
entityData = new EntityData(entityId, latestMap, null);
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData)));
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription));
}
}
@ -163,7 +159,7 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx<EntityDataQuery> {
Map<String, TsValue[]> tsMap = new HashMap<>();
tsUpdate.forEach((key, tsValue) -> tsMap.put(key, tsValue.toArray(new TsValue[tsValue.size()])));
entityData = new EntityData(entityId, null, tsMap);
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData)));
wsService.sendWsMsg(sessionId, new EntityDataUpdate(cmdId, null, Collections.singletonList(entityData), maxEntitiesPerDataSubscription));
}
}
@ -207,7 +203,7 @@ public class TbEntityDataSubCtx extends TbAbstractDataSubCtx<EntityDataQuery> {
);
}
}
wsService.sendWsMsg(sessionRef.getSessionId(), new EntityDataUpdate(cmdId, data, null));
wsService.sendWsMsg(sessionRef.getSessionId(), new EntityDataUpdate(cmdId, data, null, maxEntitiesPerDataSubscription));
subIdsToCancel.forEach(subId -> localSubscriptionService.cancelSubscription(getSessionId(), subId));
subsToAdd.forEach(localSubscriptionService::addSubscription);
}

96
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -18,11 +18,14 @@ package org.thingsboard.server.service.telemetry;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -36,6 +39,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionChangeEvent;
@ -47,14 +51,20 @@ import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import java.util.stream.Collectors;
/**
* Created by ashvayka on 27.03.18.
@ -65,16 +75,19 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
private final AttributesService attrService;
private final TimeseriesService tsService;
private final EntityViewService entityViewService;
private ExecutorService tsCallBackExecutor;
public DefaultTelemetrySubscriptionService(AttributesService attrService,
TimeseriesService tsService,
EntityViewService entityViewService,
TbClusterService clusterService,
PartitionService partitionService) {
super(clusterService, partitionService);
this.attrService = attrService;
this.tsService = tsService;
this.entityViewService = entityViewService;
}
@PostConstruct
@ -106,6 +119,54 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
ListenableFuture<List<Void>> saveFuture = tsService.save(tenantId, entityId, ts, ttl);
addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) {
Futures.addCallback(this.entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId),
new FutureCallback<List<EntityView>>() {
@Override
public void onSuccess(@Nullable List<EntityView> result) {
if (result != null) {
Map<String, List<TsKvEntry>> tsMap = new HashMap<>();
for (TsKvEntry entry : ts) {
tsMap.computeIfAbsent(entry.getKey(), s -> new ArrayList<>()).add(entry);
}
for (EntityView entityView : result) {
List<String> keys = entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null ?
entityView.getKeys().getTimeseries() : new ArrayList<>(tsMap.keySet());
List<TsKvEntry> entityViewLatest = new ArrayList<>();
long startTs = entityView.getStartTimeMs();
long endTs = entityView.getEndTimeMs() == 0 ? Long.MAX_VALUE : entityView.getEndTimeMs();
for (String key : keys) {
List<TsKvEntry> entries = tsMap.get(key);
if (entries != null) {
Optional<TsKvEntry> tsKvEntry = entries.stream()
.filter(entry -> entry.getTs() > startTs && entry.getTs() <= endTs)
.max(Comparator.comparingLong(TsKvEntry::getTs));
if (tsKvEntry.isPresent()) {
entityViewLatest.add(tsKvEntry.get());
}
}
}
if (!entityViewLatest.isEmpty()) {
saveLatestAndNotify(tenantId, entityView.getId(), entityViewLatest, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void tmp) {
}
@Override
public void onFailure(Throwable t) {
}
});
}
}
}
}
@Override
public void onFailure(Throwable t) {
log.error("Error while finding entity views by tenantId and entityId", t);
}
}, MoreExecutors.directExecutor());
}
}
@Override
@ -115,6 +176,13 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
addWsCallback(saveFuture, success -> onAttributesUpdate(tenantId, entityId, scope, attributes));
}
@Override
public void saveLatestAndNotify(TenantId tenantId, EntityId entityId, List<TsKvEntry> ts, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> saveFuture = tsService.saveLatest(tenantId, entityId, ts);
addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
}
@Override
public void deleteAndNotify(TenantId tenantId, EntityId entityId, String scope, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = attrService.removeAll(tenantId, entityId, scope, keys);
@ -122,6 +190,28 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
addWsCallback(deleteFuture, success -> onAttributesDelete(tenantId, entityId, scope, keys));
}
@Override
public void deleteLatest(TenantId tenantId, EntityId entityId, List<String> keys, FutureCallback<Void> callback) {
ListenableFuture<List<Void>> deleteFuture = tsService.removeLatest(tenantId, entityId, keys);
addMainCallback(deleteFuture, callback);
}
@Override
public void deleteAllLatest(TenantId tenantId, EntityId entityId, FutureCallback<Collection<String>> callback) {
ListenableFuture<Collection<String>> deleteFuture = tsService.removeAllLatest(tenantId, entityId);
Futures.addCallback(deleteFuture, new FutureCallback<Collection<String>>() {
@Override
public void onSuccess(@Nullable Collection<String> result) {
callback.onSuccess(result);
}
@Override
public void onFailure(Throwable t) {
callback.onFailure(t);
}
}, tsCallBackExecutor);
}
@Override
public void saveAttrAndNotify(TenantId tenantId, EntityId entityId, String scope, String key, long value, FutureCallback<Void> callback) {
saveAndNotify(tenantId, entityId, scope, Collections.singletonList(new BaseAttributeKvEntry(new LongDataEntry(key, value)
@ -188,10 +278,10 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
}
}
private void addMainCallback(ListenableFuture<List<Void>> saveFuture, final FutureCallback<Void> callback) {
Futures.addCallback(saveFuture, new FutureCallback<List<Void>>() {
private <S, R> void addMainCallback(ListenableFuture<S> saveFuture, final FutureCallback<R> callback) {
Futures.addCallback(saveFuture, new FutureCallback<S>() {
@Override
public void onSuccess(@Nullable List<Void> result) {
public void onSuccess(@Nullable S result) {
callback.onSuccess(null);
}

7
application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/EntityDataUpdate.java

@ -18,6 +18,7 @@ package org.thingsboard.server.service.telemetry.cmd.v2;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.query.EntityData;
import org.thingsboard.server.service.telemetry.sub.SubscriptionErrorCode;
@ -27,8 +28,12 @@ import java.util.List;
public class EntityDataUpdate extends DataUpdate<EntityData> {
public EntityDataUpdate(int cmdId, PageData<EntityData> data, List<EntityData> update) {
@Getter
private long allowedEntities;
public EntityDataUpdate(int cmdId, PageData<EntityData> data, List<EntityData> update, long allowedEntities) {
super(cmdId, data, update, SubscriptionErrorCode.NO_ERROR.getCode(), null);
this.allowedEntities = allowedEntities;
}
public EntityDataUpdate(int cmdId, int errorCode, String errorMsg) {

20
application/src/main/resources/thingsboard.yml

@ -50,7 +50,8 @@ server:
refresh_interval: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_INTERVAL_SEC:60}"
refresh_pool_size: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_REFRESH_POOL_SIZE:1}"
max_per_user: "${TB_SERVER_WS_DYNAMIC_PAGE_LINK_MAX_PER_USER:10}"
max_entities_per_alarm_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_ALARM_SUBSCRIPTION:1000}"
max_entities_per_data_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_DATA_SUBSCRIPTION:10000}"
max_entities_per_alarm_subscription: "${TB_SERVER_WS_MAX_ENTITIES_PER_ALARM_SUBSCRIPTION:10000}"
rest:
limits:
tenant:
@ -219,7 +220,8 @@ sql:
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"
# Specify whether to log database queries and their parameters generated by entity query repository
log_entity_queries: "${SQL_LOG_ENTITY_QUERIES:false}"
log_queries: "${SQL_LOG_QUERIES:false}"
log_queries_threshold: "${SQL_LOG_QUERIES_THRESHOLD:5000}"
postgres:
# Specify partitioning size for timestamp key-value storage. Example: DAYS, MONTHS, YEARS, INDEFINITE.
ts_key_value_partitioning: "${SQL_POSTGRES_TS_KV_PARTITIONING:MONTHS}"
@ -494,6 +496,8 @@ transport:
max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}"
client_side_rpc:
timeout: "${CLIENT_SIDE_RPC_TIMEOUT:60000}"
# Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property)
api_enabled: "${TB_TRANSPORT_API_ENABLED:true}"
# Local HTTP transport parameters
http:
enabled: "${HTTP_ENABLED:true}"
@ -561,6 +565,13 @@ queue:
max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}"
max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}"
fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}"
use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}"
confluent:
ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}"
sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}"
sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}"
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
@ -568,6 +579,7 @@ queue:
notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}"
js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"
secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}"
region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}"
@ -717,6 +729,10 @@ service:
metrics:
# Enable/disable actuator metrics.
enabled: "${METRICS_ENABLED:false}"
timer:
# Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,).
percentiles: "${METRICS_TIMER_PERCENTILES:0.5}"
management:
endpoints:

2
common/actor/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/dao-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

6
common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java

@ -40,5 +40,11 @@ public interface TimeseriesService {
ListenableFuture<List<Void>> save(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry, long ttl);
ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntry);
ListenableFuture<List<Void>> remove(TenantId tenantId, EntityId entityId, List<DeleteTsKvQuery> queries);
ListenableFuture<List<Void>> removeLatest(TenantId tenantId, EntityId entityId, Collection<String> keys);
ListenableFuture<Collection<String>> removeAllLatest(TenantId tenantId, EntityId entityId);
}

2
common/data/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/message/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.common.msg;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.Builder;
@ -51,6 +52,7 @@ public final class TbMsg implements Serializable {
private final RuleChainId ruleChainId;
private final RuleNodeId ruleNodeId;
//This field is not serialized because we use queues and there is no need to do it
@JsonIgnore
transient private final TbMsgCallback callback;
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {

2
common/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>common</artifactId>

2
common/queue/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

42
common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java

@ -16,10 +16,13 @@
package org.thingsboard.server.queue.kafka;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.CommonClientConfigs;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.List;
@ -30,6 +33,7 @@ import java.util.Properties;
*/
@Slf4j
@ConditionalOnExpression("'${queue.type:null}'=='kafka'")
@ConfigurationProperties(prefix = "queue.kafka")
@Component
public class TbKafkaSettings {
@ -65,20 +69,44 @@ public class TbKafkaSettings {
@Value("${queue.kafka.fetch_max_bytes:134217728}")
@Getter
private int fetchMaxBytes;
private int fetchMaxBytes;
@Value("${kafka.other:#{null}}")
@Value("${queue.kafka.use_confluent_cloud:false}")
private boolean useConfluent;
@Value("${queue.kafka.confluent.ssl.algorithm}")
private String sslAlgorithm;
@Value("${queue.kafka.confluent.sasl.mechanism}")
private String saslMechanism;
@Value("${queue.kafka.confluent.sasl.config}")
private String saslConfig;
@Value("${queue.kafka.confluent.security.protocol}")
private String securityProtocol;
@Setter
private List<TbKafkaProperty> other;
public Properties toProps() {
Properties props = new Properties();
props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
props.put(ProducerConfig.ACKS_CONFIG, acks);
props.put(ProducerConfig.RETRIES_CONFIG, retries);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
if(other != null){
if (useConfluent) {
props.put("ssl.endpoint.identification.algorithm", sslAlgorithm);
props.put("sasl.mechanism", saslMechanism);
props.put("sasl.jaas.config", saslConfig);
props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol);
} else {
props.put(ProducerConfig.ACKS_CONFIG, acks);
props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize);
props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs);
props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory);
}
if (other != null) {
other.forEach(kv -> props.put(kv.getKey(), kv.getValue()));
}
return props;

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java

@ -42,7 +42,7 @@ import org.thingsboard.server.queue.sqs.TbAwsSqsSettings;
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java

@ -37,7 +37,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings;
import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='in-memory' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java

@ -43,7 +43,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory {

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java

@ -43,7 +43,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class PubSubTransportQueueFactory implements TbTransportQueueFactory {

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java

@ -42,7 +42,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;

2
common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java

@ -38,7 +38,7 @@ import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSetting
import javax.annotation.PreDestroy;
@Component
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && ('${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport')")
@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && (('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport')")
@Slf4j
public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory {
private final TbQueueTransportApiSettings transportApiSettings;

13
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java

@ -16,8 +16,10 @@
package org.thingsboard.server.queue.sqs;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.CreateQueueRequest;
@ -37,9 +39,16 @@ public class TbAwsSqsAdmin implements TbQueueAdmin {
public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map<String, String> attributes) {
this.attributes = attributes;
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
AWSCredentialsProvider credentialsProvider;
if (sqsSettings.getUseDefaultCredentialProviderChain()) {
credentialsProvider = new DefaultAWSCredentialsProviderChain();
} else {
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
}
sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(new AWSStaticCredentialsProvider(awsCredentials))
.withCredentials(credentialsProvider)
.withRegion(sqsSettings.getRegion())
.build();

16
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java

@ -16,8 +16,10 @@
package org.thingsboard.server.queue.sqs;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
@ -67,13 +69,19 @@ public class TbAwsSqsConsumerTemplate<T extends TbQueueMsg> extends AbstractPara
this.decoder = decoder;
this.sqsSettings = sqsSettings;
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials);
AWSCredentialsProvider credentialsProvider;
if (sqsSettings.getUseDefaultCredentialProviderChain()) {
credentialsProvider = new DefaultAWSCredentialsProviderChain();
} else {
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
}
this.sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(credProvider)
sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(credentialsProvider)
.withRegion(sqsSettings.getRegion())
.build();
}
@Override

16
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java

@ -16,8 +16,10 @@
package org.thingsboard.server.queue.sqs;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.SendMessageRequest;
@ -54,14 +56,18 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
this.admin = admin;
this.defaultTopic = defaultTopic;
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials);
AWSCredentialsProvider credentialsProvider;
if (sqsSettings.getUseDefaultCredentialProviderChain()) {
credentialsProvider = new DefaultAWSCredentialsProviderChain();
} else {
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
}
this.sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(credProvider)
sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(credentialsProvider)
.withRegion(sqsSettings.getRegion())
.build();
producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
}

3
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java

@ -27,6 +27,9 @@ import org.springframework.stereotype.Component;
@Data
public class TbAwsSqsSettings {
@Value("${queue.aws_sqs.use_default_credential_provider_chain}")
private Boolean useDefaultCredentialProviderChain;
@Value("${queue.aws_sqs.access_key_id}")
private String accessKeyId;

2
common/stats/pom.xml

@ -22,7 +22,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

37
common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java

@ -15,13 +15,13 @@
*/
package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.annotation.PostConstruct;
import java.util.concurrent.atomic.AtomicInteger;
@Service
@ -40,6 +40,23 @@ public class DefaultStatsFactory implements StatsFactory {
@Value("${metrics.enabled:false}")
private Boolean metricsEnabled;
@Value("${metrics.timer.percentiles:0.5}")
private String timerPercentilesStr;
private double[] timerPercentiles;
@PostConstruct
public void init() {
if (!StringUtils.isEmpty(timerPercentilesStr)) {
String[] split = timerPercentilesStr.split(",");
timerPercentiles = new double[split.length];
for (int i = 0; i < split.length; i++) {
timerPercentiles[i] = Double.parseDouble(split[i]);
}
}
}
@Override
public StatsCounter createStatsCounter(String key, String statsName) {
return new StatsCounter(
@ -74,9 +91,21 @@ public class DefaultStatsFactory implements StatsFactory {
return new DefaultMessagesStats(totalCounter, successfulCounter, failedCounter);
}
@Override
public Timer createTimer(String key, String... tags) {
Timer.Builder timerBuilder = Timer.builder(key)
.tags(tags)
.publishPercentiles();
if (timerPercentiles != null && timerPercentiles.length > 0) {
timerBuilder.publishPercentiles(timerPercentiles);
}
return timerBuilder.register(meterRegistry);
}
private static class StubCounter implements Counter {
@Override
public void increment(double amount) {}
public void increment(double amount) {
}
@Override
public double count() {

4
common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.common.stats;
import io.micrometer.core.instrument.Timer;
public interface StatsFactory {
StatsCounter createStatsCounter(String key, String statsName);
@ -23,4 +25,6 @@ public interface StatsFactory {
<T extends Number> T createGauge(String key, T number, String... tags);
MessagesStats createMessagesStats(String key);
Timer createTimer(String key, String... tags);
}

2
common/transport/coap/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportContext.java

@ -28,7 +28,7 @@ import org.thingsboard.server.transport.coap.adaptors.CoapTransportAdaptor;
* Created by ashvayka on 18.10.18.
*/
@Slf4j
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')")
@Component
public class CoapTransportContext extends TransportContext {

2
common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java

@ -30,7 +30,7 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@Service("CoapTransportService")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.coap.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')")
@Slf4j
public class CoapTransportService {

2
common/transport/http/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java

@ -62,7 +62,7 @@ import java.util.function.Consumer;
* @author Andrew Shvayka
*/
@RestController
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
@RequestMapping("/api/v1")
@Slf4j
public class DeviceApiController {

2
common/transport/http/src/main/java/org/thingsboard/server/transport/http/HttpTransportContext.java

@ -29,7 +29,7 @@ import javax.annotation.PostConstruct;
* Created by ashvayka on 04.10.18.
*/
@Slf4j
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.http.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')")
@Component
public class HttpTransportContext extends TransportContext {

2
common/transport/mqtt/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java

@ -53,7 +53,7 @@ import java.util.concurrent.TimeUnit;
*/
@Slf4j
@Component("MqttSslHandlerProvider")
@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')")
@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')")
@ConditionalOnProperty(prefix = "transport.mqtt.ssl", value = "enabled", havingValue = "true", matchIfMissing = false)
public class MqttSslHandlerProvider {

2
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java

@ -31,7 +31,7 @@ import org.thingsboard.server.transport.mqtt.adaptors.MqttTransportAdaptor;
*/
@Slf4j
@Component
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')")
public class MqttTransportContext extends TransportContext {
@Getter

2
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java

@ -36,7 +36,7 @@ import javax.annotation.PreDestroy;
* @author Andrew Shvayka
*/
@Service("MqttTransportService")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.mqtt.enabled}'=='true')")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')")
@Slf4j
public class MqttTransportService {

1
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java

@ -207,6 +207,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor {
try {
return new JsonParser().parse(payload);
} catch (JsonSyntaxException ex) {
log.warn("Payload is in incorrect format: {}", payload);
throw new AdaptorException(ex);
}
}

2
common/transport/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/transport/transport-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java

@ -35,7 +35,7 @@ import java.util.concurrent.Executors;
@Slf4j
@Data
@Service
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || '${service.type:null}'=='monolith'")
@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true')")
public abstract class TransportContext {
protected final ObjectMapper mapper = new ObjectMapper();

2
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -81,7 +81,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
@Slf4j
@Service
@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-transport'")
@ConditionalOnExpression("('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport'")
public class DefaultTransportService implements TransportService {
@Value("${transport.rate_limits.enabled}")

2
common/util/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
dao/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>dao</artifactId>

8
dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java

@ -391,8 +391,12 @@ public class BaseAlarmService extends AbstractEntityService implements AlarmServ
}
private Set<EntityId> getPropagationEntityIds(Alarm alarm) {
List<EntityRelation> relations = relationService.findByTo(alarm.getTenantId(), alarm.getId(), RelationTypeGroup.ALARM);
return relations.stream().map(EntityRelation::getFrom).collect(Collectors.toSet());
if (alarm.isPropagate()) {
List<EntityRelation> relations = relationService.findByTo(alarm.getTenantId(), alarm.getId(), RelationTypeGroup.ALARM);
return relations.stream().map(EntityRelation::getFrom).collect(Collectors.toSet());
} else {
return Collections.singleton(alarm.getOriginator());
}
}
private void createAlarmRelation(TenantId tenantId, EntityId entityId, EntityId alarmId) {

8
dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.settings;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -60,6 +61,13 @@ public class AdminSettingsServiceImpl implements AdminSettingsService {
public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) {
log.trace("Executing saveAdminSettings [{}]", adminSettings);
adminSettingsValidator.validate(adminSettings, data -> tenantId);
if (adminSettings.getKey().equals("mail") && "".equals(adminSettings.getJsonValue().get("password").asText())) {
AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail");
if (mailSettings != null) {
((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText());
}
}
return adminSettingsDao.save(tenantId, adminSettings);
}

30
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java

@ -43,6 +43,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -114,12 +115,12 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository {
protected final NamedParameterJdbcTemplate jdbcTemplate;
private final TransactionTemplate transactionTemplate;
@Value("${sql.log_entity_queries:false}")
private boolean logSqlQueries;
private final DefaultQueryLogComponent queryLog;
public DefaultAlarmQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) {
public DefaultAlarmQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate, DefaultQueryLogComponent queryLog) {
this.jdbcTemplate = jdbcTemplate;
this.transactionTemplate = transactionTemplate;
this.queryLog = queryLog;
}
@Override
@ -230,8 +231,17 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository {
if (!textSearchQuery.isEmpty()) {
mainQuery = String.format("select * from (%s) a WHERE %s", mainQuery, textSearchQuery);
}
String countQuery = mainQuery;
int totalElements = jdbcTemplate.queryForObject(String.format("select count(*) from (%s) result", countQuery), ctx, Integer.class);
String countQuery = String.format("select count(*) from (%s) result", mainQuery);
long queryTs = System.currentTimeMillis();
int totalElements;
try {
totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - queryTs);
}
if (totalElements == 0) {
return AlarmDataAdapter.createAlarmData(pageLink, Collections.emptyList(), totalElements, orderedEntityIds);
}
String dataQuery = mainQuery + sortPart;
@ -239,10 +249,12 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository {
if (pageLink.getPageSize() > 0) {
dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
}
List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx);
if (logSqlQueries) {
log.info("QUERY: {}", dataQuery);
Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
queryTs = System.currentTimeMillis();
List<Map<String, Object>> rows;
try {
rows = jdbcTemplate.queryForList(dataQuery, ctx);
} finally {
queryLog.logQuery(ctx, dataQuery, System.currentTimeMillis() - queryTs);
}
return AlarmDataAdapter.createAlarmData(pageLink, rows, totalElements, orderedEntityIds);
});

88
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java

@ -17,12 +17,8 @@ package org.thingsboard.server.dao.sql.query;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.support.TransactionCallback;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.CustomerId;
@ -189,6 +185,22 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
" WHEN entity.entity_type = 'ENTITY_VIEW'" +
" THEN (select name from entity_view where id = entity_id)" +
" END as label";
private static final String SELECT_ADDITIONAL_INFO = " CASE" +
" WHEN entity.entity_type = 'TENANT'" +
" THEN (select additional_info from tenant where id = entity_id)" +
" WHEN entity.entity_type = 'CUSTOMER' " +
" THEN (select additional_info from customer where id = entity_id)" +
" WHEN entity.entity_type = 'USER'" +
" THEN (select additional_info from tb_user where id = entity_id)" +
" WHEN entity.entity_type = 'DASHBOARD'" +
" THEN (select '' from dashboard where id = entity_id)" +
" WHEN entity.entity_type = 'ASSET'" +
" THEN (select additional_info from asset where id = entity_id)" +
" WHEN entity.entity_type = 'DEVICE'" +
" THEN (select additional_info from device where id = entity_id)" +
" WHEN entity.entity_type = 'ENTITY_VIEW'" +
" THEN (select additional_info from entity_view where id = entity_id)" +
" END as additional_info";
static {
entityTableMap.put(EntityType.ASSET, "asset");
@ -221,13 +233,12 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
private final NamedParameterJdbcTemplate jdbcTemplate;
private final TransactionTemplate transactionTemplate;
private final DefaultQueryLogComponent queryLog;
@Value("${sql.log_entity_queries:false}")
private boolean logSqlQueries;
public DefaultEntityQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate) {
public DefaultEntityQueryRepository(NamedParameterJdbcTemplate jdbcTemplate, TransactionTemplate transactionTemplate, DefaultQueryLogComponent queryLog) {
this.jdbcTemplate = jdbcTemplate;
this.transactionTemplate = transactionTemplate;
this.queryLog = queryLog;
}
@Override
@ -238,11 +249,14 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
ctx.append(addEntityTableQuery(ctx, query.getEntityFilter()));
ctx.append(" e where ");
ctx.append(buildEntityWhere(ctx, query.getEntityFilter(), Collections.emptyList()));
if (logSqlQueries) {
log.info("QUERY: {}", ctx.getQuery());
Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
}
return transactionTemplate.execute(status -> jdbcTemplate.queryForObject(ctx.getQuery(), ctx, Long.class));
return transactionTemplate.execute(status -> {
long startTs = System.currentTimeMillis();
try {
return jdbcTemplate.queryForObject(ctx.getQuery(), ctx, Long.class);
} finally {
queryLog.logQuery(ctx, ctx.getQuery(), System.currentTimeMillis() - startTs);
}
});
}
@Override
@ -313,8 +327,18 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
fromClauseCount = fromClauseData;
}
String countQuery = String.format("select count(id) %s", fromClauseCount);
int totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
long startTs = System.currentTimeMillis();
int totalElements;
try {
totalElements = jdbcTemplate.queryForObject(countQuery, ctx, Integer.class);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs);
}
if (totalElements == 0) {
return new PageData<>();
}
String dataQuery = String.format("select * %s", fromClauseData);
EntityDataSortOrder sortOrder = pageLink.getSortOrder();
@ -335,11 +359,13 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
if (pageLink.getPageSize() > 0) {
dataQuery = String.format("%s limit %s offset %s", dataQuery, pageLink.getPageSize(), startIndex);
}
if (logSqlQueries) {
log.info("QUERY: {}", dataQuery);
Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {}->{}", param, ctx.getValue(param)));
startTs = System.currentTimeMillis();
List<Map<String, Object>> rows;
try {
rows = jdbcTemplate.queryForList(dataQuery, ctx);
} finally {
queryLog.logQuery(ctx, countQuery, System.currentTimeMillis() - startTs);
}
List<Map<String, Object>> rows = jdbcTemplate.queryForList(dataQuery, ctx);
return EntityDataAdapter.createEntityData(pageLink, selectionMapping, rows, totalElements);
});
}
@ -434,14 +460,16 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
//TODO: fetch last level only.
//TODO: fetch distinct records.
String lvlFilter = getLvlFilter(entityFilter.getMaxLevel());
String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, label FROM " + entityType.name() + " WHERE id in ( SELECT entity_id";
String selectFields = "SELECT tenant_id, customer_id, id, created_time, type, name, additional_info "
+ (entityType.equals(EntityType.ENTITY_VIEW) ? "" : ", label ")
+ "FROM " + entityType.name() + " WHERE id in ( SELECT entity_id";
String from = getQueryTemplate(entityFilter.getDirection());
String whereFilter = " WHERE";
if (!StringUtils.isEmpty(entityFilter.getRelationType())) {
ctx.addStringParameter("where_relation_type", entityFilter.getRelationType());
whereFilter += " re.relation_type = :where_relation_type AND";
}
whereFilter += " re.to_type = :where_entity_type";
whereFilter += " re." + (entityFilter.getDirection().equals(EntitySearchDirection.FROM) ? "to" : "from") + "_type = :where_entity_type";
from = String.format(from, lvlFilter, whereFilter);
String query = "( " + selectFields + from + ")";
@ -465,7 +493,7 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
+ SELECT_TYPE + ", " + SELECT_NAME + ", " + SELECT_LABEL + ", " +
SELECT_FIRST_NAME + ", " + SELECT_LAST_NAME + ", " + SELECT_EMAIL + ", " + SELECT_REGION + ", " +
SELECT_TITLE + ", " + SELECT_COUNTRY + ", " + SELECT_STATE + ", " + SELECT_CITY + ", " +
SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE +
SELECT_ADDRESS + ", " + SELECT_ADDRESS_2 + ", " + SELECT_ZIP + ", " + SELECT_PHONE + ", " + SELECT_ADDITIONAL_INFO +
", entity.entity_type as entity_type";
String from = getQueryTemplate(entityFilter.getDirection());
ctx.addUuidParameter("relation_root_id", rootId.getId());
@ -550,14 +578,16 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository {
private String buildTextSearchQuery(QueryContext ctx, List<EntityKeyMapping> selectionMapping, String searchText) {
if (!StringUtils.isEmpty(searchText) && !selectionMapping.isEmpty()) {
String lowerSearchText = searchText.toLowerCase() + "%";
List<String> searchPredicates = selectionMapping.stream().map(mapping -> {
String paramName = mapping.getValueAlias() + "_lowerSearchText";
ctx.addStringParameter(paramName, lowerSearchText);
return String.format("LOWER(%s) LIKE concat('%%', :%s, '%%')", mapping.getValueAlias(), paramName);
}
).collect(Collectors.toList());
return String.format(" WHERE %s", String.join(" or ", searchPredicates));
String lowerSearchText = "%" + searchText.toLowerCase() + "%";
ctx.addStringParameter("lowerSearchTextParam", lowerSearchText);
List<String> searchAliases = selectionMapping.stream().filter(EntityKeyMapping::isSearchable).map(EntityKeyMapping::getValueAlias).collect(Collectors.toList());
String searchAliasesExpression;
if (searchAliases.size() > 1) {
searchAliasesExpression = "CONCAT(" + String.join(" , ", searchAliases) + ")";
} else {
searchAliasesExpression = searchAliases.get(0);
}
return String.format(" WHERE LOWER(%s) LIKE :%s", searchAliasesExpression, "lowerSearchTextParam");
} else {
return "";
}

40
dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultQueryLogComponent.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.query;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.util.Arrays;
@Component
@Slf4j
public class DefaultQueryLogComponent implements QueryLogComponent {
@Value("${sql.log_queries:false}")
private boolean logSqlQueries;
@Value("${sql.log_queries_threshold:5000}")
private long logQueriesThreshold;
@Override
public void logQuery(QueryContext ctx, String query, long duration) {
if (logSqlQueries && duration > logQueriesThreshold) {
log.info("QUERY: {} took {}ms", query, duration);
Arrays.asList(ctx.getParameterNames()).forEach(param -> log.info("QUERY PARAM: {} -> {}", param, ctx.getValue(param)));
}
}
}

7
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityDataAdapter.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.sql.query;
import org.apache.commons.lang3.math.NumberUtils;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.UUIDConverter;
import org.thingsboard.server.common.data.id.EntityId;
@ -64,7 +65,7 @@ public class EntityDataAdapter {
String strValue;
long ts;
if (entityKey.getType().equals(EntityKeyType.ENTITY_FIELD)) {
strValue = value != null ? value.toString() : null;
strValue = value != null ? value.toString() : "";
ts = System.currentTimeMillis();
} else {
strValue = convertValue(value);
@ -82,7 +83,7 @@ public class EntityDataAdapter {
if (value != null) {
String strVal = value.toString();
// check number
if (strVal.length() > 0) {
if (strVal.length() > 0 && NumberUtils.isParsable(strVal)) {
try {
long longVal = Long.parseLong(strVal);
return Long.toString(longVal);
@ -98,7 +99,7 @@ public class EntityDataAdapter {
}
return strVal;
} else {
return null;
return "";
}
}

29
dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java

@ -71,31 +71,34 @@ public class EntityKeyMapping {
private static final String ADDRESS_2 = "address2";
private static final String ZIP = "zip";
private static final String PHONE = "phone";
private static final String ADDITIONAL_INFO = "additionalInfo";
public static final List<String> commonEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME);
public static final List<String> typedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, ADDITIONAL_INFO);
public static final List<String> widgetEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME);
public static final List<String> commonEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, ADDITIONAL_INFO);
public static final List<String> dashboardEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, TITLE);
public static final List<String> labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL);
public static final List<String> contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE);
public static final List<String> labeledEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, ADDITIONAL_INFO);
public static final List<String> contactBasedEntityFields = Arrays.asList(CREATED_TIME, ENTITY_TYPE, EMAIL, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO);
public static final Set<String> commonEntityFieldsSet = new HashSet<>(commonEntityFields);
public static final Set<String> relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE));
public static final Set<String> relationQueryEntityFieldsSet = new HashSet<>(Arrays.asList(CREATED_TIME, ENTITY_TYPE, NAME, TYPE, LABEL, FIRST_NAME, LAST_NAME, EMAIL, REGION, TITLE, COUNTRY, STATE, CITY, ADDRESS, ADDRESS_2, ZIP, PHONE, ADDITIONAL_INFO));
static {
allowedEntityFieldMap.put(EntityType.DEVICE, new HashSet<>(labeledEntityFields));
allowedEntityFieldMap.put(EntityType.ASSET, new HashSet<>(labeledEntityFields));
allowedEntityFieldMap.put(EntityType.ENTITY_VIEW, new HashSet<>(labeledEntityFields));
allowedEntityFieldMap.put(EntityType.ENTITY_VIEW, new HashSet<>(typedEntityFields));
allowedEntityFieldMap.put(EntityType.TENANT, new HashSet<>(contactBasedEntityFields));
allowedEntityFieldMap.get(EntityType.TENANT).add(REGION);
allowedEntityFieldMap.put(EntityType.CUSTOMER, new HashSet<>(contactBasedEntityFields));
allowedEntityFieldMap.put(EntityType.USER, new HashSet<>(Arrays.asList(CREATED_TIME, FIRST_NAME, LAST_NAME, EMAIL)));
allowedEntityFieldMap.put(EntityType.USER, new HashSet<>(Arrays.asList(CREATED_TIME, FIRST_NAME, LAST_NAME, EMAIL, ADDITIONAL_INFO)));
allowedEntityFieldMap.put(EntityType.DASHBOARD, new HashSet<>(dashboardEntityFields));
allowedEntityFieldMap.put(EntityType.RULE_CHAIN, new HashSet<>(commonEntityFields));
allowedEntityFieldMap.put(EntityType.RULE_NODE, new HashSet<>(commonEntityFields));
allowedEntityFieldMap.put(EntityType.WIDGET_TYPE, new HashSet<>(commonEntityFields));
allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(commonEntityFields));
allowedEntityFieldMap.put(EntityType.WIDGET_TYPE, new HashSet<>(widgetEntityFields));
allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(widgetEntityFields));
entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY);
entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY);
@ -114,6 +117,7 @@ public class EntityKeyMapping {
entityFieldColumnMap.put(ADDRESS_2, ModelConstants.ADDRESS2_PROPERTY);
entityFieldColumnMap.put(ZIP, ModelConstants.ZIP_PROPERTY);
entityFieldColumnMap.put(PHONE, ModelConstants.PHONE_PROPERTY);
entityFieldColumnMap.put(ADDITIONAL_INFO, ModelConstants.ADDITIONAL_INFO_PROPERTY);
Map<String, String> contactBasedAliases = new HashMap<>();
contactBasedAliases.put(NAME, TITLE);
@ -139,6 +143,7 @@ public class EntityKeyMapping {
private String alias;
private boolean isLatest;
private boolean isSelection;
private boolean isSearchable;
private boolean isSortOrder;
private boolean ignore = false;
private List<KeyFilter> keyFilters;
@ -303,6 +308,7 @@ public class EntityKeyMapping {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setLatest(false);
mapping.setSelection(true);
mapping.setSearchable(!key.getKey().equals(ADDITIONAL_INFO));
mapping.setEntityKey(key);
return mapping;
}
@ -311,6 +317,7 @@ public class EntityKeyMapping {
key -> {
EntityKeyMapping mapping = new EntityKeyMapping();
mapping.setLatest(true);
mapping.setSearchable(true);
mapping.setSelection(true);
mapping.setEntityKey(key);
return mapping;
@ -425,11 +432,15 @@ public class EntityKeyMapping {
private String buildComplexPredicateQuery(QueryContext ctx, String alias, EntityKey key,
ComplexFilterPredicate predicate, EntityFilterType filterType) {
return predicate.getPredicates().stream()
String result = predicate.getPredicates().stream()
.map(keyFilterPredicate -> this.buildPredicateQuery(ctx, alias, key, keyFilterPredicate, filterType))
.filter(Objects::nonNull).collect(Collectors.joining(
" " + predicate.getOperation().name() + " "
));
if (!result.trim().isEmpty()) {
result = "( " + result + " )";
}
return result;
}
private String buildSimplePredicateQuery(QueryContext ctx, String alias, EntityKey key,

21
dao/src/main/java/org/thingsboard/server/dao/sql/query/QueryLogComponent.java

@ -0,0 +1,21 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.query;
public interface QueryLogComponent {
void logQuery(QueryContext ctx, String query, long duration);
}

76
dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.timeseries;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
@ -28,6 +29,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
@ -38,6 +40,7 @@ import org.thingsboard.server.dao.service.Validator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
@ -71,9 +74,11 @@ public class BaseTimeseriesService implements TimeseriesService {
queries.forEach(this::validate);
if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
EntityView entityView = entityViewService.findEntityViewById(tenantId, (EntityViewId) entityId);
List<String> keys = entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null ?
entityView.getKeys().getTimeseries() : Collections.emptyList();
List<ReadTsKvQuery> filteredQueries =
queries.stream()
.filter(query -> entityView.getKeys().getTimeseries().isEmpty() || entityView.getKeys().getTimeseries().contains(query.getKey()))
.filter(query -> keys.isEmpty() || keys.contains(query.getKey()))
.collect(Collectors.toList());
return timeseriesDao.findAllAsync(tenantId, entityView.getEntityId(), updateQueriesForEntityView(entityView, filteredQueries));
}
@ -85,27 +90,6 @@ public class BaseTimeseriesService implements TimeseriesService {
validate(entityId);
List<ListenableFuture<TsKvEntry>> futures = Lists.newArrayListWithExpectedSize(keys.size());
keys.forEach(key -> Validator.validateString(key, "Incorrect key " + key));
if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
EntityView entityView = entityViewService.findEntityViewById(tenantId, (EntityViewId) entityId);
List<String> filteredKeys = new ArrayList<>(keys);
if (entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null &&
!entityView.getKeys().getTimeseries().isEmpty()) {
filteredKeys.retainAll(entityView.getKeys().getTimeseries());
}
List<ReadTsKvQuery> queries =
filteredKeys.stream()
.map(key -> {
long endTs = entityView.getEndTimeMs() != 0 ? entityView.getEndTimeMs() : Long.MAX_VALUE;
return new BaseReadTsKvQuery(key, entityView.getStartTimeMs(), endTs, 1, "DESC");
})
.collect(Collectors.toList());
if (queries.size() > 0) {
return timeseriesDao.findAllAsync(tenantId, entityView.getEntityId(), queries);
} else {
return Futures.immediateFuture(new ArrayList<>());
}
}
keys.forEach(key -> futures.add(timeseriesLatestDao.findLatest(tenantId, entityId, key)));
return Futures.allAsList(futures);
}
@ -113,17 +97,7 @@ public class BaseTimeseriesService implements TimeseriesService {
@Override
public ListenableFuture<List<TsKvEntry>> findAllLatest(TenantId tenantId, EntityId entityId) {
validate(entityId);
if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
EntityView entityView = entityViewService.findEntityViewById(tenantId, (EntityViewId) entityId);
if (entityView.getKeys() != null && entityView.getKeys().getTimeseries() != null &&
!entityView.getKeys().getTimeseries().isEmpty()) {
return findLatest(tenantId, entityId, entityView.getKeys().getTimeseries());
} else {
return Futures.immediateFuture(new ArrayList<>());
}
} else {
return timeseriesLatestDao.findAllLatest(tenantId, entityId);
}
return timeseriesLatestDao.findAllLatest(tenantId, entityId);
}
@Override
@ -149,6 +123,18 @@ public class BaseTimeseriesService implements TimeseriesService {
return Futures.allAsList(futures);
}
@Override
public ListenableFuture<List<Void>> saveLatest(TenantId tenantId, EntityId entityId, List<TsKvEntry> tsKvEntries) {
List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(tsKvEntries.size());
for (TsKvEntry tsKvEntry : tsKvEntries) {
if (tsKvEntry == null) {
throw new IncorrectParameterException("Key value entry can't be null");
}
futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry));
}
return Futures.allAsList(futures);
}
private void saveAndRegisterFutures(TenantId tenantId, List<ListenableFuture<Void>> futures, EntityId entityId, TsKvEntry tsKvEntry, long ttl) {
if (entityId.getEntityType().equals(EntityType.ENTITY_VIEW)) {
throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only");
@ -188,6 +174,30 @@ public class BaseTimeseriesService implements TimeseriesService {
return Futures.allAsList(futures);
}
@Override
public ListenableFuture<List<Void>> removeLatest(TenantId tenantId, EntityId entityId, Collection<String> keys) {
validate(entityId);
List<ListenableFuture<Void>> futures = Lists.newArrayListWithExpectedSize(keys.size());
for (String key : keys) {
DeleteTsKvQuery query = new BaseDeleteTsKvQuery(key, 0, System.currentTimeMillis(), false);
futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query));
}
return Futures.allAsList(futures);
}
@Override
public ListenableFuture<Collection<String>> removeAllLatest(TenantId tenantId, EntityId entityId) {
validate(entityId);
return Futures.transformAsync(this.findAllLatest(tenantId, entityId), latest -> {
if (!latest.isEmpty()) {
Collection<String> keys = latest.stream().map(TsKvEntry::getKey).collect(Collectors.toList());
return Futures.transform(this.removeLatest(tenantId, entityId, keys), res -> keys, MoreExecutors.directExecutor());
} else {
return Futures.immediateFuture(Collections.emptyList());
}
}, MoreExecutors.directExecutor());
}
private void deleteAndRegisterFutures(TenantId tenantId, List<ListenableFuture<Void>> futures, EntityId entityId, DeleteTsKvQuery query) {
futures.add(timeseriesDao.remove(tenantId, entityId, query));
futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query));

14
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -108,13 +108,13 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
super.startExecutor();
if (!isInstall()) {
getFetchStmt(Aggregation.NONE, DESC_ORDER);
Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
if (partition.isPresent()) {
tsFormat = partition.get();
} else {
log.warn("Incorrect configuration of partitioning {}", partitioning);
throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
}
}
Optional<NoSqlTsPartitionDate> partition = NoSqlTsPartitionDate.parse(partitioning);
if (partition.isPresent()) {
tsFormat = partition.get();
} else {
log.warn("Incorrect configuration of partitioning {}", partitioning);
throw new RuntimeException("Failed to parse partitioning property: " + partitioning + "!");
}
}

11
dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java

@ -28,6 +28,15 @@ public class JacksonUtil {
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
public static <T> T convertValue(Object fromValue, Class<T> toValueType) {
try {
return OBJECT_MAPPER.convertValue(fromValue, toValueType);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("The given object value: "
+ fromValue + " cannot be converted to " + toValueType);
}
}
public static <T> T fromString(String string, Class<T> clazz) {
try {
return OBJECT_MAPPER.readValue(string, clazz);
@ -60,4 +69,4 @@ public class JacksonUtil {
public static <T> T clone(T value) {
return fromString(toString(value), (Class<T>) value.getClass());
}
}
}

2
dao/src/main/resources/sql/schema-entities-idx.sql

@ -20,6 +20,8 @@ CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC);
CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC);
CREATE INDEX IF NOT EXISTS idx_event_type_entity_id ON event(tenant_id, event_type, entity_type, entity_id);
CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id);

12
dao/src/main/resources/sql/schema-entities.sql

@ -20,7 +20,17 @@ CREATE TABLE IF NOT EXISTS tb_schema_settings
CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version)
);
INSERT INTO tb_schema_settings (schema_version) VALUES (3001000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 3001000;
CREATE OR REPLACE PROCEDURE insert_tb_schema_settings()
LANGUAGE plpgsql AS
$$
BEGIN
IF (SELECT COUNT(*) FROM tb_schema_settings) = 0 THEN
INSERT INTO tb_schema_settings (schema_version) VALUES (3001000);
END IF;
END;
$$;
call insert_tb_schema_settings();
CREATE TABLE IF NOT EXISTS admin_settings (
id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY,

14
dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java

@ -25,6 +25,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
@ -45,6 +46,8 @@ import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.util.DaoTestUtil;
import org.thingsboard.server.dao.util.SqlDbType;
import java.util.*;
import java.util.concurrent.ExecutionException;
@ -60,6 +63,9 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
private TenantId tenantId;
@Autowired
private JdbcTemplate template;
@Before
public void before() {
Tenant tenant = new Tenant();
@ -821,6 +827,10 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue());
}
List<String> deviceTemperatures = temperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
if (DaoTestUtil.getSqlDbType(template) == SqlDbType.H2) {
// in H2 double values are stored with E0 in the end of the string
loadedTemperatures = loadedTemperatures.stream().map(s -> s.substring(0, s.length() - 2)).collect(Collectors.toList());
}
Assert.assertEquals(deviceTemperatures, loadedTemperatures);
pageLink = new EntityDataPageLink(10, 0, null, sortOrder);
@ -848,6 +858,10 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest {
entityData.getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).collect(Collectors.toList());
List<String> deviceHighTemperatures = highTemperatures.stream().map(aDouble -> Double.toString(aDouble)).collect(Collectors.toList());
if (DaoTestUtil.getSqlDbType(template) == SqlDbType.H2) {
// in H2 double values are stored with E0 in the end of the string
loadedHighTemperatures = loadedHighTemperatures.stream().map(s -> s.substring(0, s.length() - 2)).collect(Collectors.toList());
}
Assert.assertEquals(deviceHighTemperatures, loadedHighTemperatures);
deviceService.deleteDevicesByTenantId(tenantId);

10
dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java

@ -91,10 +91,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
saveEntries(deviceId, TS);
testLatestTsAndVerify(deviceId);
EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY, DOUBLE_KEY, LONG_KEY, BOOLEAN_KEY));
testLatestTsAndVerify(entityView.getId());
}
private void testLatestTsAndVerify(EntityId entityId) throws ExecutionException, InterruptedException {
@ -141,12 +137,6 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
List<TsKvEntry> entries = tsService.findLatest(tenantId, deviceId, Collections.singleton(STRING_KEY)).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, Arrays.asList(STRING_KEY));
entries = tsService.findLatest(tenantId, entityView.getId(), Collections.singleton(STRING_KEY)).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, stringKvEntry), entries.get(0));
}
@Test

40
dao/src/test/java/org/thingsboard/server/dao/util/DaoTestUtil.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
import org.springframework.jdbc.core.JdbcTemplate;
import java.sql.DriverManager;
public class DaoTestUtil {
private static final String POSTGRES_DRIVER_CLASS = "org.postgresql.Driver";
private static final String H2_DRIVER_CLASS = "org.hsqldb.jdbc.JDBCDriver";
public static SqlDbType getSqlDbType(JdbcTemplate template){
try {
String driverName = DriverManager.getDriver(template.getDataSource().getConnection().getMetaData().getURL()).getClass().getName();
if (POSTGRES_DRIVER_CLASS.equals(driverName)) {
return SqlDbType.POSTGRES;
} else if (H2_DRIVER_CLASS.equals(driverName)) {
return SqlDbType.H2;
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}

20
dao/src/test/java/org/thingsboard/server/dao/util/SqlDbType.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.util;
public enum SqlDbType {
POSTGRES, H2;
}

5
docker/compose-utils.sh

@ -39,6 +39,9 @@ function additionalComposeQueueArgs() {
kafka)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.kafka.yml"
;;
confluent)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.confluent.yml"
;;
aws-sqs)
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml"
;;
@ -52,7 +55,7 @@ function additionalComposeQueueArgs() {
ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml"
;;
*)
echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or aws-sqs or pubsub or rabbitmq or service-bus." >&2
echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2
exit 1
esac
echo $ADDITIONAL_COMPOSE_QUEUE_ARGS

57
docker/docker-compose.confluent.yml

@ -0,0 +1,57 @@
#
# Copyright © 2016-2020 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-js-executor:
env_file:
- queue-confluent.env
tb-core1:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-core2:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-rule-engine1:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-rule-engine2:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-mqtt-transport1:
env_file:
- queue-confluent.env
tb-mqtt-transport2:
env_file:
- queue-confluent.env
tb-http-transport1:
env_file:
- queue-confluent.env
tb-http-transport2:
env_file:
- queue-confluent.env
tb-coap-transport:
env_file:
- queue-confluent.env

18
docker/queue-confluent.env

@ -0,0 +1,18 @@
TB_QUEUE_TYPE=kafka
TB_KAFKA_SERVERS=confluent.cloud:9092
TB_QUEUE_KAFKA_REPLICATION_FACTOR=3
TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD=true
TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM=https
TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM=PLAIN
TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="CLUSTER_API_KEY" password="CLUSTER_API_SECRET";
TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL=SASL_SSL
TB_QUEUE_KAFKA_CONFLUENT_USERNAME=CLUSTER_API_KEY
TB_QUEUE_KAFKA_CONFLUENT_PASSWORD=CLUSTER_API_SECRET
TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000
TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000
TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000
TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000
TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600

2
msa/black-box-tests/pom.xml

@ -21,7 +21,7 @@
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>msa</artifactId>
</parent>
<groupId>org.thingsboard.msa</groupId>

39
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java

@ -15,11 +15,15 @@
*/
package org.thingsboard.server.msa;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.cassandra.cql3.Json;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
@ -32,6 +36,7 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.SSLContexts;
import org.json.simple.JSONObject;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.rules.TestRule;
@ -42,6 +47,7 @@ import org.thingsboard.rest.client.RestClient;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
@ -52,6 +58,7 @@ import java.net.URI;
import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
@Slf4j
@ -95,6 +102,17 @@ public abstract class AbstractContainerTest {
}
};
protected Device createGatewayDevice() throws JsonProcessingException {
String isGateway = "{\"gateway\":true}";
ObjectMapper objectMapper = new ObjectMapper();
JsonNode additionalInfo = objectMapper.readTree(isGateway);
Device gatewayDeviceTemplate = new Device();
gatewayDeviceTemplate.setName("mqtt_gateway");
gatewayDeviceTemplate.setType("gateway");
gatewayDeviceTemplate.setAdditionalInfo(additionalInfo);
return restClient.saveDevice(gatewayDeviceTemplate);
}
protected Device createDevice(String name) {
return restClient.createDevice(name + RandomStringUtils.randomAlphanumeric(7), "DEFAULT");
}
@ -140,6 +158,27 @@ public abstract class AbstractContainerTest {
return expectedValue.equals(list.get(1));
}
protected JsonObject createGatewayConnectPayload(String deviceName){
JsonObject payload = new JsonObject();
payload.addProperty("device", deviceName);
return payload;
}
protected JsonObject createGatewayPayload(String deviceName, long ts){
JsonObject payload = new JsonObject();
payload.add(deviceName, createGatewayTelemetryArray(ts));
return payload;
}
protected JsonArray createGatewayTelemetryArray(long ts){
JsonArray telemetryArray = new JsonArray();
if (ts > 0)
telemetryArray.add(createPayload(ts));
else
telemetryArray.add(createPayload());
return telemetryArray;
}
protected JsonObject createPayload(long ts) {
JsonObject values = createPayload();
JsonObject payload = new JsonObject();

376
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java

@ -0,0 +1,376 @@
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.msa.connectivity;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.Unpooled;
import io.netty.handler.codec.mqtt.MqttQoS;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.http.ResponseEntity;
import org.thingsboard.mqtt.MqttClient;
import org.thingsboard.mqtt.MqttClientConfig;
import org.thingsboard.mqtt.MqttHandler;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.msa.AbstractContainerTest;
import org.thingsboard.server.msa.WsClient;
import org.thingsboard.server.msa.mapper.WsTelemetryResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Random;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@Slf4j
public class MqttGatewayClientTest extends AbstractContainerTest {
Device gatewayDevice;
MqttClient mqttClient;
Device createdDevice;
MqttMessageListener listener;
@Before
public void createGateway() throws Exception {
restClient.login("tenant@thingsboard.org", "tenant");
this.gatewayDevice = createGatewayDevice();
Optional<DeviceCredentials> gatewayDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId());
Assert.assertTrue(gatewayDeviceCredentials.isPresent());
this.listener = new MqttMessageListener();
this.mqttClient = getMqttClient(gatewayDeviceCredentials.get(), listener);
this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice);
}
@After
public void removeGateway() throws Exception {
restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.gatewayDevice.getId());
restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId());
this.listener = null;
this.mqttClient = null;
this.createdDevice = null;
}
@Test
public void telemetryUpload() throws Exception {
WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), -1).toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
Assert.assertEquals(4, actualLatestTelemetry.getData().size());
Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"),
actualLatestTelemetry.getLatestValues().keySet());
Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString()));
Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1"));
Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0)));
Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73)));
}
@Test
public void telemetryUploadWithTs() throws Exception {
long ts = 1451649600512L;
restClient.login("tenant@thingsboard.org", "tenant");
WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS);
mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), ts).toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
Assert.assertEquals(4, actualLatestTelemetry.getData().size());
Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues());
Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString()));
Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1"));
Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0)));
Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73)));
}
@Test
public void publishAttributeUpdateToServer() throws Exception {
Optional<DeviceCredentials> createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId());
Assert.assertTrue(createdDeviceCredentials.isPresent());
WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS);
JsonObject clientAttributes = new JsonObject();
clientAttributes.addProperty("attr1", "value1");
clientAttributes.addProperty("attr2", true);
clientAttributes.addProperty("attr3", 42.0);
clientAttributes.addProperty("attr4", 73);
JsonObject gatewayClientAttributes = new JsonObject();
gatewayClientAttributes.add(createdDevice.getName(), clientAttributes);
mqttClient.publish("v1/gateway/attributes", Unpooled.wrappedBuffer(gatewayClientAttributes.toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received attributes: {}", actualLatestTelemetry);
wsClient.closeBlocking();
Assert.assertEquals(4, actualLatestTelemetry.getData().size());
Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"),
actualLatestTelemetry.getLatestValues().keySet());
Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1"));
Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString()));
Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0)));
Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73)));
}
@Test
public void requestAttributeValuesFromServer() throws Exception {
WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS);
// Add a new client attribute
JsonObject clientAttributes = new JsonObject();
String clientAttributeValue = RandomStringUtils.randomAlphanumeric(8);
clientAttributes.addProperty("clientAttr", clientAttributeValue);
JsonObject gatewayClientAttributes = new JsonObject();
gatewayClientAttributes.add(createdDevice.getName(), clientAttributes);
mqttClient.publish("v1/gateway/attributes", Unpooled.wrappedBuffer(gatewayClientAttributes.toString().getBytes())).get();
WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage();
log.info("Received ws telemetry: {}", actualLatestTelemetry);
wsClient.closeBlocking();
Assert.assertEquals(1, actualLatestTelemetry.getData().size());
Assert.assertEquals(Sets.newHashSet("clientAttr"),
actualLatestTelemetry.getLatestValues().keySet());
Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue));
// Add a new shared attribute
JsonObject sharedAttributes = new JsonObject();
String sharedAttributeValue = RandomStringUtils.randomAlphanumeric(8);
sharedAttributes.addProperty("sharedAttr", sharedAttributeValue);
ResponseEntity sharedAttributesResponse = restClient.getRestTemplate()
.postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE",
mapper.readTree(sharedAttributes.toString()), ResponseEntity.class,
createdDevice.getId());
Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful());
MqttEvent sharedAttributeEvent = listener.getEvents().poll(10, TimeUnit.SECONDS);
// Catch attribute update event
Assert.assertNotNull(sharedAttributeEvent);
Assert.assertEquals("v1/gateway/attributes", sharedAttributeEvent.getTopic());
// Subscribe to attributes response
mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
checkAttribute(true, clientAttributeValue);
checkAttribute(false, sharedAttributeValue);
}
@Test
public void subscribeToAttributeUpdatesFromServer() throws Exception {
mqttClient.on("v1/gateway/attributes", listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
String sharedAttributeName = "sharedAttr";
// Add a new shared attribute
JsonObject sharedAttributes = new JsonObject();
String sharedAttributeValue = RandomStringUtils.randomAlphanumeric(8);
sharedAttributes.addProperty(sharedAttributeName, sharedAttributeValue);
JsonObject gatewaySharedAttributeValue = new JsonObject();
gatewaySharedAttributeValue.addProperty("device", createdDevice.getName());
gatewaySharedAttributeValue.add("data", sharedAttributes);
ResponseEntity sharedAttributesResponse = restClient.getRestTemplate()
.postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE",
mapper.readTree(sharedAttributes.toString()), ResponseEntity.class,
createdDevice.getId());
Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful());
MqttEvent event = listener.getEvents().poll(10, TimeUnit.SECONDS);
Assert.assertEquals(sharedAttributeValue,
mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText());
// Update the shared attribute value
JsonObject updatedSharedAttributes = new JsonObject();
String updatedSharedAttributeValue = RandomStringUtils.randomAlphanumeric(8);
updatedSharedAttributes.addProperty(sharedAttributeName, updatedSharedAttributeValue);
JsonObject gatewayUpdatedSharedAttributeValue = new JsonObject();
gatewayUpdatedSharedAttributeValue.addProperty("device", createdDevice.getName());
gatewayUpdatedSharedAttributeValue.add("data", updatedSharedAttributes);
ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate()
.postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE",
mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class,
createdDevice.getId());
Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful());
event = listener.getEvents().poll(10, TimeUnit.SECONDS);
Assert.assertEquals(updatedSharedAttributeValue,
mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText());
}
@Test
public void serverSideRpc() throws Exception {
String gatewayRpcTopic = "v1/gateway/rpc";
mqttClient.on(gatewayRpcTopic, listener, MqttQoS.AT_LEAST_ONCE).get();
// Wait until subscription is processed
TimeUnit.SECONDS.sleep(3);
// Send an RPC from the server
JsonObject serverRpcPayload = new JsonObject();
serverRpcPayload.addProperty("method", "getValue");
serverRpcPayload.addProperty("params", true);
ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor());
ListenableFuture<ResponseEntity> future = service.submit(() -> {
try {
return restClient.getRestTemplate()
.postForEntity(HTTPS_URL + "/api/plugins/rpc/twoway/{deviceId}",
mapper.readTree(serverRpcPayload.toString()), String.class,
createdDevice.getId());
} catch (IOException e) {
return ResponseEntity.badRequest().build();
}
});
// Wait for RPC call from the server and send the response
MqttEvent requestFromServer = listener.getEvents().poll(10, TimeUnit.SECONDS);
Assert.assertNotNull(requestFromServer);
Assert.assertNotNull(requestFromServer.getMessage());
JsonObject requestFromServerJson = new JsonParser().parse(requestFromServer.getMessage()).getAsJsonObject();
Assert.assertEquals(createdDevice.getName(), requestFromServerJson.get("device").getAsString());
JsonObject requestFromServerData = requestFromServerJson.get("data").getAsJsonObject();
Assert.assertEquals("getValue", requestFromServerData.get("method").getAsString());
Assert.assertTrue(requestFromServerData.get("params").getAsBoolean());
int requestId = requestFromServerData.get("id").getAsInt();
JsonObject clientResponse = new JsonObject();
clientResponse.addProperty("response", "someResponse");
JsonObject gatewayResponse = new JsonObject();
gatewayResponse.addProperty("device", createdDevice.getName());
gatewayResponse.addProperty("id", requestId);
gatewayResponse.add("data", clientResponse);
// Send a response to the server's RPC request
mqttClient.publish(gatewayRpcTopic, Unpooled.wrappedBuffer(gatewayResponse.toString().getBytes())).get();
ResponseEntity serverResponse = future.get(5, TimeUnit.SECONDS);
Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful());
Assert.assertEquals(clientResponse.toString(), serverResponse.getBody());
}
private void checkAttribute(boolean client, String expectedValue) throws Exception{
JsonObject gatewayAttributesRequest = new JsonObject();
int messageId = new Random().nextInt(100);
gatewayAttributesRequest.addProperty("id", messageId);
gatewayAttributesRequest.addProperty("device", createdDevice.getName());
gatewayAttributesRequest.addProperty("client", client);
String attributeName;
if (client)
attributeName = "clientAttr";
else
attributeName = "sharedAttr";
gatewayAttributesRequest.addProperty("key", attributeName);
log.info(gatewayAttributesRequest.toString());
mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get();
MqttEvent clientAttributeEvent = listener.getEvents().poll(10, TimeUnit.SECONDS);
Assert.assertNotNull(clientAttributeEvent);
JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject();
Assert.assertEquals(messageId, responseMessage.get("id").getAsInt());
Assert.assertEquals(createdDevice.getName(), responseMessage.get("device").getAsString());
Assert.assertEquals(3, responseMessage.entrySet().size());
Assert.assertEquals(expectedValue, responseMessage.get("value").getAsString());
}
private Device createDeviceThroughGateway(MqttClient mqttClient, Device gatewayDevice) throws Exception {
String deviceName = "mqtt_device";
mqttClient.publish("v1/gateway/connect", Unpooled.wrappedBuffer(createGatewayConnectPayload(deviceName).toString().getBytes())).get();
TimeUnit.SECONDS.sleep(3);
List<EntityRelation> relations = restClient.findByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON);
Assert.assertEquals(1, relations.size());
EntityId createdEntityId = relations.get(0).getTo();
DeviceId createdDeviceId = new DeviceId(createdEntityId.getId());
Optional<Device> createdDevice = restClient.getDeviceById(createdDeviceId);
Assert.assertTrue(createdDevice.isPresent());
return createdDevice.get();
}
private MqttClient getMqttClient(DeviceCredentials deviceCredentials, MqttMessageListener listener) throws InterruptedException, ExecutionException {
MqttClientConfig clientConfig = new MqttClientConfig();
clientConfig.setClientId("MQTT client from test");
clientConfig.setUsername(deviceCredentials.getCredentialsId());
MqttClient mqttClient = MqttClient.create(clientConfig, listener);
mqttClient.connect("localhost", 1883).get();
return mqttClient;
}
@Data
private class MqttMessageListener implements MqttHandler {
private final BlockingQueue<MqttEvent> events;
private MqttMessageListener() {
events = new ArrayBlockingQueue<>(100);
}
@Override
public void onMessage(String topic, ByteBuf message) {
log.info("MQTT message [{}], topic [{}]", message.toString(StandardCharsets.UTF_8), topic);
events.add(new MqttEvent(topic, message.toString(StandardCharsets.UTF_8)));
}
}
@Data
private class MqttEvent {
private final String topic;
private final String message;
}
}

6
msa/js-executor/config/custom-environment-variables.yml

@ -26,6 +26,12 @@ kafka:
servers: "TB_KAFKA_SERVERS"
replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR"
topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES"
use_confluent_cloud: "TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD"
confluent:
sasl:
mechanism: "TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM"
username: "TB_QUEUE_KAFKA_CONFLUENT_USERNAME"
password: "TB_QUEUE_KAFKA_CONFLUENT_PASSWORD"
pubsub:
project_id: "TB_QUEUE_PUBSUB_PROJECT_ID"

4
msa/js-executor/config/default.yml

@ -26,6 +26,10 @@ kafka:
servers: "localhost:9092"
replication_factor: "1"
topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600"
use_confluent_cloud: false
confluent:
sasl:
mechanism: "PLAIN"
pubsub:
queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800"

1
msa/js-executor/docker/Dockerfile

@ -22,6 +22,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/start-js-executor.sh /usr/bin
RUN yes | dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

2
msa/js-executor/package-lock.json

@ -1,6 +1,6 @@
{
"name": "thingsboard-js-executor",
"version": "3.1.0",
"version": "3.1.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

2
msa/js-executor/package.json

@ -1,7 +1,7 @@
{
"name": "thingsboard-js-executor",
"private": true,
"version": "3.1.0",
"version": "3.1.1",
"description": "ThingsBoard JavaScript Executor Microservice",
"main": "server.js",
"bin": "server.js",

2
msa/js-executor/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>msa</artifactId>
</parent>
<groupId>org.thingsboard.msa</groupId>

20
msa/js-executor/queue/kafkaTemplate.js

@ -61,15 +61,27 @@ function KafkaProducer() {
const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');
const requestTopic = config.get('request_topic');
const useConfluent = config.get('kafka.use_confluent_cloud');
logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
logger.info('Kafka Requests Topic: %s', requestTopic);
kafkaClient = new Kafka({
let kafkaConfig = {
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
});
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
};
if (useConfluent) {
kafkaConfig['sasl'] = {
mechanism: config.get('kafka.confluent.sasl.mechanism'),
username: config.get('kafka.confluent.username'),
password: config.get('kafka.confluent.password')
};
kafkaConfig['ssl'] = true;
}
kafkaClient = new Kafka(kafkaConfig);
parseTopicProperties();

2
msa/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>msa</artifactId>

1
msa/tb-node/docker/Dockerfile

@ -22,6 +22,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/start-tb-node.sh /usr/bin
RUN yes | dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

2
msa/tb-node/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>msa</artifactId>
</parent>
<groupId>org.thingsboard.msa</groupId>

1
msa/tb/docker-cassandra/Dockerfile

@ -39,6 +39,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/stop-db.sh /usr/bin
RUN dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

1
msa/tb/docker-postgres/Dockerfile

@ -35,6 +35,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/stop-db.sh /usr/bin
RUN dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

1
msa/tb/docker-tb/Dockerfile

@ -26,6 +26,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/stop-db.sh /usr/bin
RUN dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

2
msa/tb/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>msa</artifactId>
</parent>
<groupId>org.thingsboard.msa</groupId>

1
msa/transport/coap/docker/Dockerfile

@ -22,6 +22,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/start-tb-coap-transport.sh /usr/bin
RUN yes | dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

2
msa/transport/coap/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.msa</groupId>
<version>3.1.0-SNAPSHOT</version>
<version>3.1.1-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.msa.transport</groupId>

1
msa/transport/http/docker/Dockerfile

@ -22,6 +22,7 @@ RUN chmod a+x /tmp/*.sh \
&& mv /tmp/start-tb-http-transport.sh /usr/bin
RUN yes | dpkg -i /tmp/${pkg.name}.deb
RUN rm /tmp/${pkg.name}.deb
RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save