{
}
@Override
- public DataUpdateType getDataUpdateType() {
- return DataUpdateType.ENTITY_DATA;
+ public CmdUpdateType getCmdUpdateType() {
+ return CmdUpdateType.ENTITY_DATA;
}
@JsonCreator
diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/UnsubscribeCmd.java b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/UnsubscribeCmd.java
index d34f5c6fdf..5e046ee581 100644
--- a/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/UnsubscribeCmd.java
+++ b/application/src/main/java/org/thingsboard/server/service/telemetry/cmd/v2/UnsubscribeCmd.java
@@ -15,8 +15,6 @@
*/
package org.thingsboard.server.service.telemetry.cmd.v2;
-import lombok.Data;
-
public interface UnsubscribeCmd {
int getCmdId();
diff --git a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java
index d542bdbe61..74d81c0307 100644
--- a/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java
+++ b/application/src/main/java/org/thingsboard/server/service/transport/msg/TransportToDeviceActorMsgWrapper.java
@@ -34,6 +34,8 @@ import java.util.UUID;
@Data
public class TransportToDeviceActorMsgWrapper implements TbActorMsg, DeviceAwareMsg, TenantAwareMsg, Serializable {
+ private static final long serialVersionUID = 7191333353202935941L;
+
private final TenantId tenantId;
private final DeviceId deviceId;
private final TransportToDeviceActorMsg msg;
diff --git a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java b/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java
new file mode 100644
index 0000000000..4ce0d2f3d5
--- /dev/null
+++ b/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java
@@ -0,0 +1,85 @@
+/**
+ * Copyright © 2016-2021 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.utils;
+
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
+
+/**
+ * This class deduplicate executions of the specified function.
+ * Useful in cluster mode, when you get event about partition change multiple times.
+ * Assuming that the function execution is expensive, we should execute it immediately when first time event occurs and
+ * later, once the processing of first event is done, process last pending task.
+ *
+ * @param parameters of the function
+ */
+@Slf4j
+public class EventDeduplicationExecutor
{
+ private final String name;
+ private final ExecutorService executor;
+ private final Consumer
function;
+ private P pendingTask;
+ private boolean busy;
+
+ public EventDeduplicationExecutor(String name, ExecutorService executor, Consumer
function) {
+ this.name = name;
+ this.executor = executor;
+ this.function = function;
+ }
+
+ public void submit(P params) {
+ log.info("[{}] Going to submit: {}", name, params);
+ synchronized (EventDeduplicationExecutor.this) {
+ if (!busy) {
+ busy = true;
+ pendingTask = null;
+ try {
+ log.info("[{}] Submitting task: {}", name, params);
+ executor.submit(() -> {
+ try {
+ log.info("[{}] Executing task: {}", name, params);
+ function.accept(params);
+ } catch (Throwable e) {
+ log.warn("[{}] Failed to process task with parameters: {}", name, params, e);
+ throw e;
+ } finally {
+ unlockAndProcessIfAny();
+ }
+ });
+ } catch (Throwable e) {
+ log.warn("[{}] Failed to submit task with parameters: {}", name, params, e);
+ unlockAndProcessIfAny();
+ throw e;
+ }
+ } else {
+ log.info("[{}] Task is already in progress. {} pending task: {}", name, pendingTask == null ? "adding" : "updating", params);
+ pendingTask = params;
+ }
+ }
+ }
+
+ private void unlockAndProcessIfAny() {
+ synchronized (EventDeduplicationExecutor.this) {
+ busy = false;
+ if (pendingTask != null) {
+ submit(pendingTask);
+ }
+ }
+ }
+}
diff --git a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java
index 6ca82aa062..8ee5a5ebc7 100644
--- a/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java
+++ b/application/src/main/java/org/thingsboard/server/utils/MiscUtils.java
@@ -33,6 +33,7 @@ public class MiscUtils {
return "The " + propertyName + " property need to be set!";
}
+ @SuppressWarnings("deprecation")
public static HashFunction forName(String name) {
switch (name) {
case "murmur3_32":
diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml
index 6d10a74854..9170a96ea6 100644
--- a/application/src/main/resources/logback.xml
+++ b/application/src/main/resources/logback.xml
@@ -30,7 +30,7 @@
-
+
diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml
index 0cc7c08669..6ac41979d6 100644
--- a/application/src/main/resources/thingsboard.yml
+++ b/application/src/main/resources/thingsboard.yml
@@ -34,6 +34,7 @@ server:
log_controller_error_stack_trace: "${HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE:false}"
ws:
send_timeout: "${TB_SERVER_WS_SEND_TIMEOUT:5000}"
+ ping_timeout: "${TB_SERVER_WS_PING_TIMEOUT:30000}"
limits:
# Limit the amount of sessions and subscriptions available on each server. Put values to zero to disable particular limitation
max_sessions_per_tenant: "${TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_TENANT:0}"
@@ -109,7 +110,7 @@ security:
# Enable/disable claiming devices, if false -> the device's [claimingAllowed] SERVER_SCOPE attribute must be set to [true] to allow claiming specific device
allowClaimingByDefault: "${SECURITY_CLAIM_ALLOW_CLAIMING_BY_DEFAULT:true}"
# Time allowed to claim the device in milliseconds
- duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value
+ duration: "${SECURITY_CLAIM_DURATION:86400000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value
basic:
enabled: "${SECURITY_BASIC_ENABLED:false}"
oauth2:
@@ -118,6 +119,15 @@ security:
githubMapper:
emailUrl: "${SECURITY_OAUTH2_GITHUB_MAPPER_EMAIL_URL_KEY:https://api.github.com/user/emails}"
+# Usage statistics parameters
+usage:
+ stats:
+ report:
+ enabled: "${USAGE_STATS_REPORT_ENABLED:true}"
+ interval: "${USAGE_STATS_REPORT_INTERVAL:10}"
+ check:
+ cycle: "${USAGE_STATS_CHECK_CYCLE:60000}"
+
# Dashboard parameters
dashboard:
# Maximum allowed datapoints fetched by widgets
@@ -313,6 +323,9 @@ actors:
cache:
# caffeine or redis
type: "${CACHE_TYPE:caffeine}"
+ attributes:
+ # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random'
+ enabled: "${CACHE_ATTRIBUTES_ENABLED:true}"
caffeine:
specs:
@@ -335,8 +348,8 @@ caffeine:
timeToLiveInMinutes: 1440
maxSize: 0
claimDevices:
- timeToLiveInMinutes: 1
- maxSize: 0
+ timeToLiveInMinutes: 1440
+ maxSize: 1000
securitySettings:
timeToLiveInMinutes: 1440
maxSize: 0
@@ -346,6 +359,9 @@ caffeine:
deviceProfiles:
timeToLiveInMinutes: 1440
maxSize: 0
+ attributes:
+ timeToLiveInMinutes: 1440
+ maxSize: 100000
redis:
# standalone or cluster
@@ -530,6 +546,7 @@ transport:
http:
enabled: "${HTTP_ENABLED:true}"
request_timeout: "${HTTP_REQUEST_TIMEOUT:60000}"
+ max_request_timeout: "${HTTP_MAX_REQUEST_TIMEOUT:300000}"
# Local MQTT transport parameters
mqtt:
# Enable/disable mqtt transport protocol.
@@ -608,11 +625,15 @@ queue:
security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}"
other:
topic-properties:
- rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
- core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
- transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
- notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}"
- js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}"
+ rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+ core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+ transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+ notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+ js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}"
+ consumer-stats:
+ enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
+ print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}"
+ kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}"
aws_sqs:
use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}"
access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"
diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
index 12f4e5fced..e25592e35e 100644
--- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
+++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
@@ -375,6 +375,10 @@ public abstract class AbstractWebTest {
return readResponse(doGetAsync(urlTemplate, urlVariables).andExpect(status().isOk()), responseClass);
}
+ protected T doGetAsyncTyped(String urlTemplate, TypeReference responseType, Object... urlVariables) throws Exception {
+ return readResponse(doGetAsync(urlTemplate, urlVariables).andExpect(status().isOk()), responseType);
+ }
+
protected ResultActions doGetAsync(String urlTemplate, Object... urlVariables) throws Exception {
MockHttpServletRequestBuilder getRequest;
getRequest = get(urlTemplate, urlVariables);
diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java
index d4e4b4e0d6..5d234d2691 100644
--- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java
+++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityQueryControllerTest.java
@@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.EntityListFilter;
+import org.thingsboard.server.common.data.query.EntityTypeFilter;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.query.KeyFilter;
import org.thingsboard.server.common.data.query.NumericFilterPredicate;
@@ -132,6 +133,14 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe
count = doPostWithResponse("/api/entitiesQuery/count", countQuery, Long.class);
Assert.assertEquals(97, count.longValue());
+
+ EntityTypeFilter filter2 = new EntityTypeFilter();
+ filter2.setEntityType(EntityType.DEVICE);
+
+ EntityCountQuery countQuery2 = new EntityCountQuery(filter2);
+
+ Long count2 = doPostWithResponse("/api/entitiesQuery/count", countQuery2, Long.class);
+ Assert.assertEquals(97, count2.longValue());
}
@Test
@@ -198,11 +207,31 @@ public abstract class BaseEntityQueryControllerTest extends AbstractControllerTe
Assert.assertEquals(11, data.getTotalElements());
Assert.assertEquals("Device19", data.getData().get(0).getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue());
+
+ EntityTypeFilter filter2 = new EntityTypeFilter();
+ filter2.setEntityType(EntityType.DEVICE);
+
+ EntityDataSortOrder sortOrder2 = new EntityDataSortOrder(
+ new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC
+ );
+ EntityDataPageLink pageLink2 = new EntityDataPageLink(10, 0, null, sortOrder2);
+ List entityFields2 = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name"));
+
+ EntityDataQuery query2 = new EntityDataQuery(filter2, pageLink2, entityFields2, null, null);
+
+ PageData data2 =
+ doPostWithTypedResponse("/api/entitiesQuery/find", query2, new TypeReference>() {
+ });
+
+ Assert.assertEquals(97, data2.getTotalElements());
+ Assert.assertEquals(10, data2.getTotalPages());
+ Assert.assertTrue(data2.hasNext());
+ Assert.assertEquals(10, data2.getData().size());
+
}
@Test
public void testFindEntityDataByQueryWithAttributes() throws Exception {
-
List devices = new ArrayList<>();
List temperatures = new ArrayList<>();
List highTemperatures = new ArrayList<>();
diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
index 35ff3703c0..c6d0073d83 100644
--- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
+++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java
@@ -347,8 +347,8 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes
Thread.sleep(1000);
- List