Browse Source

Merge pull request #7001 from thingsboard/feature/events-optimization

Events Optimization
pull/7025/head
Andrew Shvayka 4 years ago
committed by GitHub
parent
commit
0678dbef18
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 230
      application/src/main/data/upgrade/3.4.0/schema_update.sql
  2. 159
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  3. 18
      application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java
  4. 2
      application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java
  5. 70
      application/src/main/java/org/thingsboard/server/controller/EventController.java
  6. 9
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  7. 4
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  8. 14
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  9. 13
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  10. 29
      application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java
  11. 13
      application/src/main/resources/thingsboard.yml
  12. 3
      application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java
  13. 17
      application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java
  14. 19
      application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java
  15. 19
      application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java
  16. 18
      common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java
  17. 6
      common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java
  18. 8
      common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java
  19. 47
      common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java
  20. 26
      common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java
  21. 66
      common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java
  22. 2
      common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java
  23. 71
      common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java
  24. 6
      common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java
  25. 26
      common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java
  26. 2
      common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java
  27. 74
      common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java
  28. 63
      common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java
  29. 17
      common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java
  30. 102
      common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java
  31. 47
      common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java
  32. 60
      common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java
  33. 16
      common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java
  34. 121
      dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java
  35. 67
      dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java
  36. 31
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  37. 1
      dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java
  38. 1
      dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java
  39. 64
      dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java
  40. 116
      dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java
  41. 69
      dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java
  42. 63
      dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java
  43. 109
      dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java
  44. 64
      dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java
  45. 12
      dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java
  46. 114
      dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java
  47. 3
      dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java
  48. 199
      dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java
  49. 48
      dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java
  50. 217
      dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java
  51. 402
      dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java
  52. 118
      dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java
  53. 117
      dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java
  54. 153
      dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java
  55. 161
      dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java
  56. 120
      dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java
  57. 4
      dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java
  58. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java
  59. 10
      dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java
  60. 24
      dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql
  61. 68
      dao/src/main/resources/sql/schema-entities.sql
  62. 20
      dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java
  63. 68
      dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java
  64. 167
      dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java
  65. 2
      dao/src/test/resources/cassandra-test.properties
  66. 10
      rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java
  67. 2
      ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts
  68. 45
      ui-ngx/src/app/modules/home/components/event/event-table-config.ts
  69. 9
      ui-ngx/src/app/shared/models/event.models.ts
  70. 3
      ui-ngx/src/assets/locale/locale.constant-en_US.json

230
application/src/main/data/upgrade/3.4.0/schema_update.sql

@ -0,0 +1,230 @@
--
-- Copyright © 2016-2022 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
CREATE TABLE IF NOT EXISTS rule_node_debug_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL ,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar,
e_type varchar,
e_entity_id uuid,
e_entity_type varchar,
e_msg_id uuid,
e_msg_type varchar,
e_data_type varchar,
e_relation_type varchar,
e_data varchar,
e_metadata varchar,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS rule_chain_debug_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_message varchar,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS stats_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_messages_processed bigint NOT NULL,
e_errors_occurred bigint NOT NULL
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS lc_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_type varchar NOT NULL,
e_success boolean NOT NULL,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS error_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_method varchar NOT NULL,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main
ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main
ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_stats_event_main
ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_lc_event_main
ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_error_event_main
ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE OR REPLACE FUNCTION to_safe_json(p_json text) RETURNS json
LANGUAGE plpgsql AS
$$
BEGIN
return REPLACE(p_json, '\u0000', '' )::json;
EXCEPTION
WHEN OTHERS THEN
return '{}'::json;
END;
$$;
-- Useful to migrate old events to the new table structure;
CREATE OR REPLACE PROCEDURE migrate_regular_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int)
LANGUAGE plpgsql AS
$$
DECLARE
partition_size_in_ms bigint;
p record;
table_name varchar;
BEGIN
partition_size_in_ms = partition_size_in_hours * 3600 * 1000;
FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('STATS', 'LC_EVENT', 'ERROR') and ts >= start_ts_in_ms and ts < end_ts_in_ms
LOOP
IF p.event_type = 'STATS' THEN
table_name := 'stats_event';
ELSEIF p.event_type = 'LC_EVENT' THEN
table_name := 'lc_event';
ELSEIF p.event_type = 'ERROR' THEN
table_name := 'error_event';
END IF;
RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms);
EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms));
END LOOP;
INSERT INTO stats_event
SELECT id,
tenant_id,
ts,
entity_id,
body ->> 'server',
(body ->> 'messagesProcessed')::bigint,
(body ->> 'errorsOccurred')::bigint
FROM
(select id, tenant_id, ts, entity_id, to_safe_json(body) as body
FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'STATS' AND to_safe_json(body) ->> 'server' IS NOT NULL
) safe_event
ON CONFLICT DO NOTHING;
INSERT INTO lc_event
SELECT id,
tenant_id,
ts,
entity_id,
body ->> 'server',
body ->> 'event',
(body ->> 'success')::boolean,
body ->> 'error'
FROM
(select id, tenant_id, ts, entity_id, to_safe_json(body) as body
FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'LC_EVENT' AND to_safe_json(body) ->> 'server' IS NOT NULL
) safe_event
ON CONFLICT DO NOTHING;
INSERT INTO error_event
SELECT id,
tenant_id,
ts,
entity_id,
body ->> 'server',
body ->> 'method',
body ->> 'error'
FROM
(select id, tenant_id, ts, entity_id, to_safe_json(body) as body
FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'ERROR' AND to_safe_json(body) ->> 'server' IS NOT NULL
) safe_event
ON CONFLICT DO NOTHING;
END
$$;
-- Useful to migrate old debug events to the new table structure;
CREATE OR REPLACE PROCEDURE migrate_debug_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int)
LANGUAGE plpgsql AS
$$
DECLARE
partition_size_in_ms bigint;
p record;
table_name varchar;
BEGIN
partition_size_in_ms = partition_size_in_hours * 3600 * 1000;
FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') and ts >= start_ts_in_ms and ts < end_ts_in_ms
LOOP
IF p.event_type = 'DEBUG_RULE_NODE' THEN
table_name := 'rule_node_debug_event';
ELSEIF p.event_type = 'DEBUG_RULE_CHAIN' THEN
table_name := 'rule_chain_debug_event';
END IF;
RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms);
EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms));
END LOOP;
INSERT INTO rule_node_debug_event
SELECT id,
tenant_id,
ts,
entity_id,
body ->> 'server',
body ->> 'type',
(body ->> 'entityId')::uuid,
body ->> 'entityName',
(body ->> 'msgId')::uuid,
body ->> 'msgType',
body ->> 'dataType',
body ->> 'relationType',
body ->> 'data',
body ->> 'metadata',
body ->> 'error'
FROM
(select id, tenant_id, ts, entity_id, to_safe_json(body) as body
FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_NODE' AND to_safe_json(body) ->> 'server' IS NOT NULL
) safe_event
ON CONFLICT DO NOTHING;
INSERT INTO rule_chain_debug_event
SELECT id,
tenant_id,
ts,
entity_id,
body ->> 'server',
body ->> 'message',
body ->> 'error'
FROM
(select id, tenant_id, ts, entity_id, to_safe_json(body) as body
FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_CHAIN' AND to_safe_json(body) ->> 'server' IS NOT NULL
) safe_event
ON CONFLICT DO NOTHING;
END
$$;

159
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -37,10 +37,11 @@ import org.thingsboard.rule.engine.api.sms.SmsSenderFactory;
import org.thingsboard.server.actors.service.ActorService;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbActorMsg;
@ -112,6 +113,29 @@ import java.util.concurrent.TimeUnit;
@Component
public class ActorSystemContext {
private static final FutureCallback<Void> RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Void event) {
}
@Override
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Rule Chain", th);
}
};
private static final FutureCallback<Void> RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() {
@Override
public void onSuccess(@Nullable Void event) {
}
@Override
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Node", th);
}
};
protected final ObjectMapper mapper = new ObjectMapper();
private final ConcurrentMap<TenantId, DebugTbRateLimits> debugPerTenantLimits = new ConcurrentHashMap<>();
@ -462,25 +486,28 @@ public class ActorSystemContext {
}
public void persistError(TenantId tenantId, EntityId entityId, String method, Exception e) {
Event event = new Event();
event.setTenantId(tenantId);
event.setEntityId(entityId);
event.setType(DataConstants.ERROR);
event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e)));
persistEvent(event);
eventService.saveAsync(ErrorEvent.builder()
.tenantId(tenantId)
.entityId(entityId.getId())
.serviceId(getServiceId())
.method(method)
.error(toString(e)).build());
}
public void persistLifecycleEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent lcEvent, Exception e) {
Event event = new Event();
event.setTenantId(tenantId);
event.setEntityId(entityId);
event.setType(DataConstants.LC_EVENT);
event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e)));
persistEvent(event);
}
LifecycleEvent.LifecycleEventBuilder event = LifecycleEvent.builder()
.tenantId(tenantId)
.entityId(entityId.getId())
.serviceId(getServiceId())
.lcEventType(lcEvent.name());
if (e != null) {
event.success(false).error(toString(e));
} else {
event.success(true);
}
private void persistEvent(Event event) {
eventService.saveAsync(event);
eventService.saveAsync(event.build());
}
private String toString(Throwable e) {
@ -489,21 +516,6 @@ public class ActorSystemContext {
return sw.toString();
}
private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional<Exception> e) {
ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name());
if (e.isPresent()) {
node = node.put("success", false);
node = node.put("error", toString(e.get()));
} else {
node = node.put("success", true);
}
return node;
}
private JsonNode toBodyJson(String serviceId, String method, String body) {
return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body);
}
public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) {
return partitionService.resolve(serviceType, tenantId, entityId);
}
@ -539,44 +551,27 @@ public class ActorSystemContext {
private void persistDebugAsync(TenantId tenantId, EntityId entityId, String type, TbMsg tbMsg, String relationType, Throwable error, String failureMessage) {
if (checkLimits(tenantId, tbMsg, error)) {
try {
Event event = new Event();
event.setTenantId(tenantId);
event.setEntityId(entityId);
event.setType(DataConstants.DEBUG_RULE_NODE);
String metadata = mapper.writeValueAsString(tbMsg.getMetaData().getData());
ObjectNode node = mapper.createObjectNode()
.put("type", type)
.put("server", getServiceId())
.put("entityId", tbMsg.getOriginator().getId().toString())
.put("entityName", tbMsg.getOriginator().getEntityType().name())
.put("msgId", tbMsg.getId().toString())
.put("msgType", tbMsg.getType())
.put("dataType", tbMsg.getDataType().name())
.put("relationType", relationType)
.put("data", tbMsg.getData())
.put("metadata", metadata);
RuleNodeDebugEvent.RuleNodeDebugEventBuilder event = RuleNodeDebugEvent.builder()
.tenantId(tenantId)
.entityId(entityId.getId())
.serviceId(getServiceId())
.eventType(type)
.eventEntity(tbMsg.getOriginator())
.msgId(tbMsg.getId())
.msgType(tbMsg.getType())
.dataType(tbMsg.getDataType().name())
.relationType(relationType)
.data(tbMsg.getData())
.metadata(mapper.writeValueAsString(tbMsg.getMetaData().getData()));
if (error != null) {
node = node.put("error", toString(error));
event.error(toString(error));
} else if (failureMessage != null) {
node = node.put("error", failureMessage);
event.error(failureMessage);
}
event.setBody(node);
ListenableFuture<Void> future = eventService.saveAsync(event);
Futures.addCallback(future, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void event) {
}
@Override
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Node", th);
}
}, MoreExecutors.directExecutor());
ListenableFuture<Void> future = eventService.saveAsync(event.build());
Futures.addCallback(future, RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
} catch (IOException ex) {
log.warn("Failed to persist rule node debug message", ex);
}
@ -603,33 +598,17 @@ public class ActorSystemContext {
}
private void persistRuleChainDebugModeEvent(TenantId tenantId, EntityId entityId, Throwable error) {
Event event = new Event();
event.setTenantId(tenantId);
event.setEntityId(entityId);
event.setType(DataConstants.DEBUG_RULE_CHAIN);
ObjectNode node = mapper.createObjectNode()
//todo: what fields are needed here?
.put("server", getServiceId())
.put("message", "Reached debug mode rate limit!");
RuleChainDebugEvent.RuleChainDebugEventBuilder event = RuleChainDebugEvent.builder()
.tenantId(tenantId)
.entityId(entityId.getId())
.serviceId(getServiceId())
.message("Reached debug mode rate limit!");
if (error != null) {
node = node.put("error", toString(error));
event.error(toString(error));
}
event.setBody(node);
ListenableFuture<Void> future = eventService.saveAsync(event);
Futures.addCallback(future, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void event) {
}
@Override
public void onFailure(Throwable th) {
log.error("Could not save debug Event for Rule Chain", th);
}
}, MoreExecutors.directExecutor());
ListenableFuture<Void> future = eventService.saveAsync(event.build());
Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor());
}
public static Exception toException(Throwable error) {

18
application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java

@ -20,13 +20,13 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.TbActor;
import org.thingsboard.server.actors.TbActorCtx;
import org.thingsboard.server.actors.TbActorId;
import org.thingsboard.server.actors.TbStringActorId;
import org.thingsboard.server.actors.service.ContextAwareActor;
import org.thingsboard.server.actors.service.ContextBasedCreator;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
@ -54,12 +54,14 @@ public class StatsActor extends ContextAwareActor {
if (msg.isEmpty()) {
return;
}
Event event = new Event();
event.setEntityId(msg.getEntityId());
event.setTenantId(msg.getTenantId());
event.setType(DataConstants.STATS);
event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred()));
systemContext.getEventService().saveAsync(event);
systemContext.getEventService().saveAsync(StatisticsEvent.builder()
.tenantId(msg.getTenantId())
.entityId(msg.getEntityId().getId())
.serviceId(systemContext.getServiceInfoProvider().getServiceId())
.messagesProcessed(msg.getMessagesProcessed())
.errorsOccurred(msg.getErrorsOccurred())
.build()
);
}
private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) {

2
application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java

@ -94,7 +94,7 @@ public class ControllerConstants {
protected static final String DEVICE_PROFILE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, transportType, description, isDefault";
protected static final String ASSET_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle";
protected static final String ALARM_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, startTs, endTs, type, ackTs, clearTs, severity, status";
protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, id";
protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "ts, id";
protected static final String EDGE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle";
protected static final String RULE_CHAIN_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, root";
protected static final String WIDGET_BUNDLE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, title, tenantId";

70
application/src/main/java/org/thingsboard/server/controller/EventController.java

@ -29,8 +29,11 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.EventFilter;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -42,6 +45,8 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.Operation;
import java.util.Locale;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE;
@ -110,7 +115,7 @@ public class EventController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/events/{entityType}/{entityId}/{eventType}", method = RequestMethod.GET)
@ResponseBody
public PageData<Event> getEvents(
public PageData<EventInfo> getEvents(
@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable(ENTITY_TYPE) String strEntityType,
@ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true)
@ -135,16 +140,12 @@ public class EventController extends BaseController {
@RequestParam(required = false) Long endTime) throws ThingsboardException {
checkParameter("EntityId", strEntityId);
checkParameter("EntityType", strEntityType);
try {
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
return checkNotNull(eventService.findEvents(tenantId, entityId, eventType, pageLink));
} catch (Exception e) {
throw handleException(e);
}
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
return checkNotNull(eventService.findEvents(tenantId, entityId, resolveEventType(eventType), pageLink));
}
@ApiOperation(value = "Get Events (getEvents)",
@ -153,7 +154,7 @@ public class EventController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.GET)
@ResponseBody
public PageData<Event> getEvents(
public PageData<EventInfo> getEvents(
@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable(ENTITY_TYPE) String strEntityType,
@ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true)
@ -176,18 +177,14 @@ public class EventController extends BaseController {
@RequestParam(required = false) Long endTime) throws ThingsboardException {
checkParameter("EntityId", strEntityId);
checkParameter("EntityType", strEntityType);
try {
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
return checkNotNull(eventService.findEvents(tenantId, entityId, pageLink));
} catch (Exception e) {
throw handleException(e);
}
return checkNotNull(eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, pageLink));
}
@ApiOperation(value = "Get Events by event filter (getEvents)",
@ -198,7 +195,7 @@ public class EventController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.POST)
@ResponseBody
public PageData<Event> getEvents(
public PageData<EventInfo> getEvents(
@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable(ENTITY_TYPE) String strEntityType,
@ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true)
@ -223,21 +220,17 @@ public class EventController extends BaseController {
@RequestParam(required = false) Long endTime) throws ThingsboardException {
checkParameter("EntityId", strEntityId);
checkParameter("EntityType", strEntityType);
try {
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId));
if (sortProperty != null && sortProperty.equals("createdTime") && eventFilter.hasFilterForJsonBody()) {
sortProperty = ModelConstants.CREATED_TIME_PROPERTY;
}
EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId);
checkEntityId(entityId, Operation.READ);
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink));
} catch (Exception e) {
throw handleException(e);
if (sortProperty != null && sortProperty.equals("createdTime")) {
sortProperty = ModelConstants.TS_COLUMN;
}
TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime);
return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink));
}
@ApiOperation(value = "Clear Events (clearEvents)", notes = "Clears events by filter for specified entity.")
@ -266,4 +259,13 @@ public class EventController extends BaseController {
}
}
private static EventType resolveEventType(String eventType) throws ThingsboardException {
for (var et : EventType.values()) {
if (et.name().equalsIgnoreCase(eventType) || et.getOldName().equalsIgnoreCase(eventType)) {
return et;
}
}
throw new ThingsboardException("Event type: '" + eventType + "' is not supported!", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
}

9
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -39,10 +39,10 @@ import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.rule.engine.api.ScriptEngine;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.actors.tenant.DebugTbRateLimits;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.RuleChainId;
@ -70,7 +70,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@ -353,10 +352,10 @@ public class RuleChainController extends BaseController {
RuleNodeId ruleNodeId = new RuleNodeId(toUUID(strRuleNodeId));
checkRuleNode(ruleNodeId, Operation.READ);
TenantId tenantId = getCurrentUser().getTenantId();
List<Event> events = eventService.findLatestEvents(tenantId, ruleNodeId, DataConstants.DEBUG_RULE_NODE, 2);
List<EventInfo> events = eventService.findLatestEvents(tenantId, ruleNodeId, EventType.DEBUG_RULE_NODE, 2);
JsonNode result = null;
if (events != null) {
for (Event event : events) {
for (EventInfo event : events) {
JsonNode body = event.getBody();
if (body.has("type") && body.get("type").asText().equals("IN")) {
result = body;

4
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -221,6 +221,10 @@ public class ThingsboardInstallService {
log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.3.4");
dataUpdateService.updateData("3.3.4");
case "3.4.0":
log.info("Upgrading ThingsBoard from version 3.4.0 to 3.4.1 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.4.0");
dataUpdateService.updateData("3.4.0");
log.info("Updating system data...");
systemDataLoaderService.updateSystemWidgets();
break;

14
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -25,6 +25,7 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.QueueId;
@ -596,6 +597,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
log.error("Failed updating schema!!!", e);
}
break;
case "3.4.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.0", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Updating schema settings...");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004001;");
log.info("Schema updated.");
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
@ -666,5 +679,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
}

13
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -58,6 +58,7 @@ import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.alarm.AlarmDao;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.model.sql.DeviceProfileEntity;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
@ -128,6 +129,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private SystemDataLoaderService systemDataLoaderService;
@Autowired
private EventService eventService;
@Override
public void updateData(String fromVersion) throws Exception {
switch (fromVersion) {
@ -159,6 +163,13 @@ public class DefaultDataUpdateService implements DataUpdateService {
tenantsProfileQueueConfigurationUpdater.updateEntities();
rateLimitsUpdater.updateEntities();
break;
case "3.4.0":
String skipEventsMigration = System.getenv("TB_SKIP_EVENTS_MIGRATION");
if (skipEventsMigration == null || skipEventsMigration.equalsIgnoreCase("false")) {
log.info("Updating data from version 3.4.0 to 3.4.1 ...");
eventService.migrateEvents();
}
break;
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
}
@ -602,7 +613,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
});
}
} catch (Exception e) {
log.error("Failed to update tenant profile queue configuration name=["+profile.getName()+"], id=["+ profile.getId().getId() +"]", e);
log.error("Failed to update tenant profile queue configuration name=[" + profile.getName() + "], id=[" + profile.getId().getId() + "]", e);
}
}

29
application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java

@ -25,7 +25,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
import java.util.concurrent.TimeUnit;
@TbCoreComponent
@Slf4j
@Service
public class EventsCleanUpService extends AbstractCleanUpService {
@ -39,9 +38,6 @@ public class EventsCleanUpService extends AbstractCleanUpService {
@Value("${sql.ttl.events.debug_events_ttl}")
private long debugTtlInSec;
@Value("${sql.ttl.events.execution_interval_ms}")
private long executionIntervalInMs;
@Value("${sql.ttl.events.enabled}")
private boolean ttlTaskExecutionEnabled;
@ -54,28 +50,11 @@ public class EventsCleanUpService extends AbstractCleanUpService {
@Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.events.execution_interval_ms}")
public void cleanUp() {
if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) {
if (ttlTaskExecutionEnabled) {
long ts = System.currentTimeMillis();
long regularEventStartTs;
long regularEventEndTs;
long debugEventStartTs;
long debugEventEndTs;
if (ttlInSec > 0) {
regularEventEndTs = ts - TimeUnit.SECONDS.toMillis(ttlInSec);
regularEventStartTs = regularEventEndTs - 2 * executionIntervalInMs;
} else {
regularEventStartTs = regularEventEndTs = 0;
}
if (debugTtlInSec > 0) {
debugEventEndTs = ts - TimeUnit.SECONDS.toMillis(debugTtlInSec);
debugEventStartTs = debugEventEndTs - 2 * executionIntervalInMs;
} else {
debugEventStartTs = debugEventEndTs = 0;
}
eventService.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs);
long regularEventExpTs = ttlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(ttlInSec) : 0;
long debugEventExpTs = debugTtlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(debugTtlInSec) : 0;
eventService.cleanupEvents(regularEventExpTs, debugEventExpTs, isSystemTenantPartitionMine());
}
}

13
application/src/main/resources/thingsboard.yml

@ -221,9 +221,6 @@ cassandra:
ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}"
ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}"
ts_key_value_ttl: "${TS_KV_TTL:0}"
events_ttl: "${TS_EVENTS_TTL:0}"
# Specify TTL of debug log in seconds. The current value corresponds to one week
debug_events_ttl: "${DEBUG_EVENTS_TTL:604800}"
buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}"
concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}"
permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}"
@ -262,6 +259,8 @@ sql:
batch_max_delay: "${SQL_EVENTS_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_EVENTS_BATCH_STATS_PRINT_MS:10000}"
batch_threads: "${SQL_EVENTS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution
partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week.
debug_partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:1}" # Number of hours to partition the debug events. The current value corresponds to one hour.
edge_events:
batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}"
batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}"
@ -287,9 +286,11 @@ sql:
ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds
events:
enabled: "${SQL_TTL_EVENTS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes
events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds
debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week
execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:3600000}" # Number of milliseconds (max random initial delay and fixed period).
# Number of seconds. TTL is disabled by default. Accuracy of the cleanup depends on the sql.events.partition_size parameter.
events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}"
# Number of seconds. The current value corresponds to one week. Accuracy of the cleanup depends on the sql.events.debug_partition_size parameter.
debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}"
edge_events:
enabled: "${SQL_TTL_EDGE_EVENTS_ENABLED:true}"
execution_interval_ms: "${SQL_TTL_EDGE_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day

3
application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java

@ -18,7 +18,8 @@ package org.thingsboard.server.actors.stats;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;

17
application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java

@ -19,8 +19,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
@ -60,20 +60,19 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle
return doGet("/api/ruleChain/metadata/" + ruleChainId.getId().toString(), RuleChainMetaData.class);
}
protected PageData<Event> getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception {
return getEvents(tenantId, entityId, DataConstants.DEBUG_RULE_NODE, limit);
protected PageData<EventInfo> getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception {
return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE.getOldName(), limit);
}
protected PageData<Event> getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception {
protected PageData<EventInfo> getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception {
TimePageLink pageLink = new TimePageLink(limit);
return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&",
new TypeReference<PageData<Event>>() {
new TypeReference<PageData<EventInfo>>() {
}, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId());
}
protected JsonNode getMetadata(Event outEvent) {
protected JsonNode getMetadata(EventInfo outEvent) {
String metaDataStr = outEvent.getBody().get("metadata").asText();
try {
return mapper.readTree(metaDataStr);
@ -82,7 +81,7 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle
}
}
protected Predicate<Event> filterByCustomEvent() {
protected Predicate<EventInfo> filterByCustomEvent() {
return event -> event.getBody().get("msgType").textValue().equals("CUSTOM");
}

19
application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java

@ -30,9 +30,10 @@ import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.page.PageData;
@ -177,15 +178,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
actorSystem.tell(qMsg);
Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
PageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
PageData<EventInfo> eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000);
List<EventInfo> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
Assert.assertEquals(2, events.size());
Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
Assert.assertEquals(ruleChain.getFirstRuleNodeId(), inEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText());
Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
Assert.assertEquals(ruleChain.getFirstRuleNodeId(), outEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText());
@ -299,16 +300,16 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess();
PageData<Event> eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000);
List<Event> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
PageData<EventInfo> eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000);
List<EventInfo> events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
Assert.assertEquals(2, events.size());
Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), inEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText());
Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), outEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText());

19
application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java

@ -20,18 +20,15 @@ import org.awaitility.Awaitility;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.test.context.support.TestPropertySourceUtils;
import org.testcontainers.containers.GenericContainer;
import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.rule.RuleChain;
@ -109,11 +106,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac
Assert.assertNotNull(ruleChainFinal.getFirstRuleNodeId());
//TODO find out why RULE_NODE update event did not appear all the time
List<Event> rcEvents = Awaitility.await("Rule Node started successfully")
List<EventInfo> rcEvents = Awaitility.await("Rule Node started successfully")
.pollInterval(10, MILLISECONDS)
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
List<Event> debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000)
List<EventInfo> debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), EventType.LC_EVENT.getOldName(), 1000)
.getData().stream().filter(e -> {
var body = e.getBody();
return body.has("event") && body.get("event").asText().equals("STARTED")
@ -145,11 +142,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac
log.warn("awaiting tbMsgCallback");
Mockito.verify(tbMsgCallback, Mockito.timeout(TimeUnit.SECONDS.toMillis(TIMEOUT))).onSuccess();
log.warn("awaiting events");
List<Event> events = Awaitility.await("get debug by custom event")
List<EventInfo> events = Awaitility.await("get debug by custom event")
.pollInterval(10, MILLISECONDS)
.atMost(TIMEOUT, TimeUnit.SECONDS)
.until(() -> {
List<Event> debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000)
List<EventInfo> debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000)
.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList());
log.warn("filtered debug events [{}]", debugEvents.size());
debugEvents.forEach((e) -> log.warn("event: {}", e));
@ -158,11 +155,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac
x -> x.size() == 2);
log.warn("asserting..");
Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get();
Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), inEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText());
Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get();
Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), outEvent.getEntityId());
Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText());

18
common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java

@ -16,34 +16,32 @@
package org.thingsboard.server.dao.event;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventFilter;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
import java.util.List;
import java.util.Optional;
public interface EventService {
ListenableFuture<Void> saveAsync(Event event);
Optional<Event> findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid);
PageData<EventInfo> findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink);
PageData<Event> findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink);
List<EventInfo> findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit);
PageData<Event> findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink);
List<Event> findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit);
PageData<Event> findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink);
PageData<EventInfo> findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink);
void removeEvents(TenantId tenantId, EntityId entityId);
void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime);
void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs);
void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb);
void migrateEvents();
}

6
common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java

@ -52,12 +52,6 @@ public class DataConstants {
}
public static final String ALARM = "ALARM";
public static final String ERROR = "ERROR";
public static final String LC_EVENT = "LC_EVENT";
public static final String STATS = "STATS";
public static final String DEBUG_RULE_NODE = "DEBUG_RULE_NODE";
public static final String DEBUG_RULE_CHAIN = "DEBUG_RULE_CHAIN";
public static final String IN = "IN";
public static final String OUT = "OUT";

8
common/data/src/main/java/org/thingsboard/server/common/data/Event.java → common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java

@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId;
*/
@Data
@ApiModel
public class Event extends BaseData<EventId> {
public class EventInfo extends BaseData<EventId> {
@ApiModelProperty(position = 1, value = "JSON object with Tenant Id.", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private TenantId tenantId;
@ -41,15 +41,15 @@ public class Event extends BaseData<EventId> {
@ApiModelProperty(position = 5, value = "Event body.", dataType = "com.fasterxml.jackson.databind.JsonNode")
private transient JsonNode body;
public Event() {
public EventInfo() {
super();
}
public Event(EventId id) {
public EventInfo(EventId id) {
super(id);
}
public Event(Event event) {
public EventInfo(EventInfo event) {
super(event);
}

47
common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java

@ -0,0 +1,47 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.springframework.data.domain.Page;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.StringUtils;
import java.util.UUID;
@Data
@ApiModel
public abstract class DebugEventFilter implements EventFilter {
@ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152")
protected String server;
@ApiModelProperty(position = 10, value = "Boolean value to filter the errors", allowableValues = "false, true")
protected boolean isError;
@ApiModelProperty(position = 11, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB")
protected String errorStr;
public void setIsError(boolean isError) {
this.isError = isError;
}
@Override
public boolean isNotEmpty() {
return !StringUtils.isEmpty(server) || isError || !StringUtils.isEmpty(errorStr);
}
}

26
common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java

@ -1,26 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import io.swagger.annotations.ApiModel;
@ApiModel
public class DebugRuleNodeEventFilter extends DebugEvent {
@Override
public EventType getEventType() {
return EventType.DEBUG_RULE_NODE;
}
}

66
common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java

@ -0,0 +1,66 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@ToString
@EqualsAndHashCode(callSuper = true)
public class ErrorEvent extends Event {
private static final long serialVersionUID = 960461434033192571L;
@Builder
private ErrorEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String method, String error) {
super(tenantId, entityId, serviceId, id, ts);
this.method = method;
this.error = error;
}
@Getter
@Setter
private String method;
@Getter
@Setter
private String error;
@Override
public EventType getType() {
return EventType.ERROR;
}
@Override
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = super.toInfo(entityType);
var json = (ObjectNode) eventInfo.getBody();
json.put("method", method);
if (error != null) {
json.put("error", error);
}
return eventInfo;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java

@ -37,7 +37,7 @@ public class ErrorEventFilter implements EventFilter {
}
@Override
public boolean hasFilterForJsonBody() {
public boolean isNotEmpty() {
return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(method) || !StringUtils.isEmpty(errorStr);
}
}

71
common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java

@ -0,0 +1,71 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.EventId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@Data
@EqualsAndHashCode(callSuper = true)
public abstract class Event extends BaseData<EventId> {
protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
protected final TenantId tenantId;
protected final UUID entityId;
protected final String serviceId;
public Event(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts) {
super();
if (id != null) {
this.id = new EventId(id);
}
this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID;
this.entityId = entityId;
this.serviceId = serviceId;
this.createdTime = ts;
}
public abstract EventType getType();
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = new EventInfo();
eventInfo.setTenantId(tenantId);
eventInfo.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId));
eventInfo.setType(getType().getOldName());
eventInfo.setId(id);
eventInfo.setUid(id.toString());
eventInfo.setCreatedTime(createdTime);
eventInfo.setBody(OBJECT_MAPPER.createObjectNode().put("server", getServiceId()));
return eventInfo;
}
protected static void putNotNull(ObjectNode json, String key, String value) {
if (value != null) {
json.put(key, value);
}
}
}

6
common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java

@ -26,8 +26,8 @@ import io.swagger.annotations.ApiModelProperty;
include = JsonTypeInfo.As.PROPERTY,
property = "eventType")
@JsonSubTypes({
@JsonSubTypes.Type(value = DebugRuleNodeEventFilter.class, name = "DEBUG_RULE_NODE"),
@JsonSubTypes.Type(value = DebugRuleChainEventFilter.class, name = "DEBUG_RULE_CHAIN"),
@JsonSubTypes.Type(value = RuleNodeDebugEventFilter.class, name = "DEBUG_RULE_NODE"),
@JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"),
@JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"),
@JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"),
@JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS")
@ -37,6 +37,6 @@ public interface EventFilter {
@ApiModelProperty(position = 1, required = true, value = "String value representing the event type", example = "STATS")
EventType getEventType();
boolean hasFilterForJsonBody();
boolean isNotEmpty();
}

26
common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java

@ -15,6 +15,30 @@
*/
package org.thingsboard.server.common.data.event;
import lombok.Getter;
public enum EventType {
ERROR, LC_EVENT, STATS, DEBUG_RULE_NODE, DEBUG_RULE_CHAIN
ERROR("error_event", "ERROR"),
LC_EVENT("lc_event", "LC_EVENT"),
STATS("stats_event", "STATS"),
DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true),
DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true);
@Getter
private final String table;
@Getter
private final String oldName;
@Getter
private final boolean debug;
EventType(String table, String oldName) {
this(table, oldName, false);
}
EventType(String table, String oldName, boolean debug) {
this.table = table;
this.oldName = oldName;
this.debug = debug;
}
}

2
common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java

@ -39,7 +39,7 @@ public class LifeCycleEventFilter implements EventFilter {
}
@Override
public boolean hasFilterForJsonBody() {
public boolean isNotEmpty() {
return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(event) || !StringUtils.isEmpty(status) || !StringUtils.isEmpty(errorStr);
}
}

74
common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java

@ -0,0 +1,74 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.jetbrains.annotations.NotNull;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@ToString
@EqualsAndHashCode(callSuper = true)
public class LifecycleEvent extends Event {
private static final long serialVersionUID = -3247420461850911549L;
@Builder
private LifecycleEvent(TenantId tenantId, UUID entityId, String serviceId,
UUID id, long ts,
String lcEventType, boolean success, String error) {
super(tenantId, entityId, serviceId, id, ts);
this.lcEventType = lcEventType;
this.success = success;
this.error = error;
}
@Getter
private final String lcEventType;
@Getter
private final boolean success;
@Getter
@Setter
private String error;
@Override
public EventType getType() {
return EventType.LC_EVENT;
}
@Override
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = super.toInfo(entityType);
var json = (ObjectNode) eventInfo.getBody();
json.put("event", lcEventType)
.put("success", success);
if (error != null) {
json.put("error", error);
}
return eventInfo;
}
}

63
common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java

@ -0,0 +1,63 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@ToString
@EqualsAndHashCode(callSuper = true)
public class RuleChainDebugEvent extends Event {
private static final long serialVersionUID = -386392236201116767L;
@Builder
private RuleChainDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String message, String error) {
super(tenantId, entityId, serviceId, id, ts);
this.message = message;
this.error = error;
}
@Getter
@Setter
private String message;
@Getter
@Setter
private String error;
@Override
public EventType getType() {
return EventType.DEBUG_RULE_CHAIN;
}
@Override
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = super.toInfo(entityType);
var json = (ObjectNode) eventInfo.getBody();
putNotNull(json, "message", message);
putNotNull(json, "error", error);
return eventInfo;
}
}

17
common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java → common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java

@ -16,11 +16,26 @@
package org.thingsboard.server.common.data.event;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.StringUtils;
@Data
@EqualsAndHashCode(callSuper = true)
@ApiModel
public class DebugRuleChainEventFilter extends DebugEvent {
public class RuleChainDebugEventFilter extends DebugEventFilter {
@ApiModelProperty(position = 2, value = "String value representing the message")
protected String message;
@Override
public EventType getEventType() {
return EventType.DEBUG_RULE_CHAIN;
}
@Override
public boolean isNotEmpty() {
return super.isNotEmpty() || !StringUtils.isEmpty(message);
}
}

102
common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java

@ -0,0 +1,102 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@ToString
@EqualsAndHashCode(callSuper = true)
public class RuleNodeDebugEvent extends Event {
private static final long serialVersionUID = -6575797430064573984L;
@Builder
private RuleNodeDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts,
String eventType, EntityId eventEntity, UUID msgId,
String msgType, String dataType, String relationType,
String data, String metadata, String error) {
super(tenantId, entityId, serviceId, id, ts);
this.eventType = eventType;
this.eventEntity = eventEntity;
this.msgId = msgId;
this.msgType = msgType;
this.dataType = dataType;
this.relationType = relationType;
this.data = data;
this.metadata = metadata;
this.error = error;
}
@Getter
private final String eventType;
@Getter
private final EntityId eventEntity;
@Getter
private final UUID msgId;
@Getter
private final String msgType;
@Getter
private final String dataType;
@Getter
private final String relationType;
@Getter
@Setter
private String data;
@Getter
@Setter
private String metadata;
@Getter
@Setter
private String error;
@Override
public EventType getType() {
return EventType.DEBUG_RULE_NODE;
}
@Override
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = super.toInfo(entityType);
var json = (ObjectNode) eventInfo.getBody();
json.put("type", eventType);
if (eventEntity != null) {
json.put("entityId", eventEntity.getId().toString())
.put("entityType", eventEntity.getEntityType().name());
}
if (msgId != null) {
json.put("msgId", msgId.toString());
}
putNotNull(json, "msgType", msgType);
putNotNull(json, "dataType", dataType);
putNotNull(json, "relationType", relationType);
putNotNull(json, "data", data);
putNotNull(json, "metadata", metadata);
putNotNull(json, "error", error);
return eventInfo;
}
}

47
common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java → common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java

@ -18,41 +18,40 @@ package org.thingsboard.server.common.data.event;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.StringUtils;
@Data
@EqualsAndHashCode(callSuper = true)
@ApiModel
public abstract class DebugEvent implements EventFilter {
public class RuleNodeDebugEventFilter extends DebugEventFilter {
@ApiModelProperty(position = 1, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT")
@ApiModelProperty(position = 2, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT")
protected String msgDirectionType;
@ApiModelProperty(position = 2, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152")
protected String server;
@ApiModelProperty(position = 3, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity")
protected String dataSearch;
@ApiModelProperty(position = 4, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName")
protected String metadataSearch;
@ApiModelProperty(position = 5, value = "String value representing the entity type", allowableValues = "DEVICE")
protected String entityName;
@ApiModelProperty(position = 6, value = "String value representing the type of message routing", example = "Success")
protected String relationType;
@ApiModelProperty(position = 7, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f")
@ApiModelProperty(position = 3, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f")
protected String entityId;
@ApiModelProperty(position = 8, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST")
@ApiModelProperty(position = 4, value = "String value representing the entity type", allowableValues = "DEVICE")
protected String entityType;
@ApiModelProperty(position = 5, value = "String value representing the message id in the rule engine", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f")
protected String msgId;
@ApiModelProperty(position = 6, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST")
protected String msgType;
@ApiModelProperty(position = 9, value = "Boolean value to filter the errors", allowableValues = "false, true")
protected boolean isError;
@ApiModelProperty(position = 10, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB")
protected String errorStr;
@ApiModelProperty(position = 7, value = "String value representing the type of message routing", example = "Success")
protected String relationType;
@ApiModelProperty(position = 8, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity")
protected String dataSearch;
@ApiModelProperty(position = 9, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName")
protected String metadataSearch;
public void setIsError(boolean isError) {
this.isError = isError;
@Override
public EventType getEventType() {
return EventType.DEBUG_RULE_NODE;
}
@Override
public boolean hasFilterForJsonBody() {
return !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(server) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch)
|| !StringUtils.isEmpty(entityName) || !StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(msgType) || !StringUtils.isEmpty(errorStr) || isError;
public boolean isNotEmpty() {
return super.isNotEmpty() || !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(entityId)
|| !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) ||
!StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch);
}
}

60
common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.ToString;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.UUID;
@ToString
@EqualsAndHashCode(callSuper = true)
public class StatisticsEvent extends Event {
private static final long serialVersionUID = 6683733979448910631L;
@Builder
private StatisticsEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, long messagesProcessed, long errorsOccurred) {
super(tenantId, entityId, serviceId, id, ts);
this.messagesProcessed = messagesProcessed;
this.errorsOccurred = errorsOccurred;
}
@Getter
private final long messagesProcessed;
@Getter
private final long errorsOccurred;
@Override
public EventType getType() {
return EventType.STATS;
}
@Override
public EventInfo toInfo(EntityType entityType) {
EventInfo eventInfo = super.toInfo(entityType);
var json = (ObjectNode) eventInfo.getBody();
json.put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred);
return eventInfo;
}
}

16
common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java

@ -27,9 +27,13 @@ public class StatisticsEventFilter implements EventFilter {
@ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152")
protected String server;
@ApiModelProperty(position = 2, value = "The minimum number of successfully processed messages", example = "25")
protected Integer messagesProcessed;
@ApiModelProperty(position = 3, value = "The minimum number of errors occurred during messages processing", example = "30")
protected Integer errorsOccurred;
protected Integer minMessagesProcessed;
@ApiModelProperty(position = 3, value = "The maximum number of successfully processed messages", example = "250")
protected Integer maxMessagesProcessed;
@ApiModelProperty(position = 4, value = "The minimum number of errors occurred during messages processing", example = "30")
protected Integer minErrorsOccurred;
@ApiModelProperty(position = 5, value = "The maximum number of errors occurred during messages processing", example = "300")
protected Integer maxErrorsOccurred;
@Override
public EventType getEventType() {
@ -37,7 +41,9 @@ public class StatisticsEventFilter implements EventFilter {
}
@Override
public boolean hasFilterForJsonBody() {
return !StringUtils.isEmpty(server) || (messagesProcessed != null && messagesProcessed > 0) || (errorsOccurred != null && errorsOccurred > 0);
public boolean isNotEmpty() {
return !StringUtils.isEmpty(server)
|| (minMessagesProcessed != null && minMessagesProcessed > 0) || (minErrorsOccurred != null && minErrorsOccurred > 0)
|| (maxMessagesProcessed != null && maxMessagesProcessed > 0) || (maxErrorsOccurred != null && maxErrorsOccurred > 0);
}
}

121
dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java

@ -15,31 +15,41 @@
*/
package org.thingsboard.server.dao.event;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventFilter;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.IdBased;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
import java.util.List;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@Service
@Slf4j
public class BaseEventService implements EventService {
@Value("${sql.ttl.events.events_ttl:0}")
private long ttlInSec;
@Value("${sql.ttl.events.debug_events_ttl:604800}")
private long debugTtlInSec;
@Value("${event.debug.max-symbols:4096}")
private int maxDebugEventSymbols;
@ -57,78 +67,85 @@ public class BaseEventService implements EventService {
}
private void checkAndTruncateDebugEvent(Event event) {
if (event.getType().startsWith("DEBUG") && event.getBody() != null && event.getBody().has("data")) {
String dataStr = event.getBody().get("data").asText();
int length = dataStr.length();
switch (event.getType()) {
case DEBUG_RULE_NODE:
RuleNodeDebugEvent rnEvent = (RuleNodeDebugEvent) event;
truncateField(rnEvent, RuleNodeDebugEvent::getData, RuleNodeDebugEvent::setData);
truncateField(rnEvent, RuleNodeDebugEvent::getMetadata, RuleNodeDebugEvent::setMetadata);
truncateField(rnEvent, RuleNodeDebugEvent::getError, RuleNodeDebugEvent::setError);
break;
case DEBUG_RULE_CHAIN:
RuleChainDebugEvent rcEvent = (RuleChainDebugEvent) event;
truncateField(rcEvent, RuleChainDebugEvent::getMessage, RuleChainDebugEvent::setMessage);
truncateField(rcEvent, RuleChainDebugEvent::getError, RuleChainDebugEvent::setError);
break;
case LC_EVENT:
LifecycleEvent lcEvent = (LifecycleEvent) event;
truncateField(lcEvent, LifecycleEvent::getError, LifecycleEvent::setError);
break;
case ERROR:
ErrorEvent eEvent = (ErrorEvent) event;
truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError);
break;
}
}
private <T extends Event> void truncateField(T event, Function<T, String> getter, BiConsumer<T, String> setter) {
var str = getter.apply(event);
if (StringUtils.isNotEmpty(str)) {
var length = str.length();
if (length > maxDebugEventSymbols) {
((ObjectNode) event.getBody()).put("data", dataStr.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]");
log.trace("[{}] Event was truncated: {}", event.getId(), dataStr);
setter.accept(event, str.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]");
}
}
}
@Override
public Optional<Event> findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) {
if (tenantId == null) {
throw new DataValidationException("Tenant id should be specified!.");
}
if (entityId == null) {
throw new DataValidationException("Entity id should be specified!.");
}
if (StringUtils.isEmpty(eventType)) {
throw new DataValidationException("Event type should be specified!.");
}
if (StringUtils.isEmpty(eventUid)) {
throw new DataValidationException("Event uid should be specified!.");
}
Event event = eventDao.findEvent(tenantId.getId(), entityId, eventType, eventUid);
return event != null ? Optional.of(event) : Optional.empty();
public PageData<EventInfo> findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink) {
return convert(entityId.getEntityType(), eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, pageLink));
}
@Override
public PageData<Event> findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink) {
return eventDao.findEvents(tenantId.getId(), entityId, pageLink);
public List<EventInfo> findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit) {
return convert(entityId.getEntityType(), eventDao.findLatestEvents(tenantId.getId(), entityId.getId(), eventType, limit));
}
@Override
public PageData<Event> findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink) {
return eventDao.findEvents(tenantId.getId(), entityId, eventType, pageLink);
public PageData<EventInfo> findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) {
return convert(entityId.getEntityType(), eventDao.findEventByFilter(tenantId.getId(), entityId.getId(), eventFilter, pageLink));
}
@Override
public List<Event> findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) {
return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit);
public void removeEvents(TenantId tenantId, EntityId entityId) {
removeEvents(tenantId, entityId, null, null, null);
}
@Override
public PageData<Event> findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) {
return eventDao.findEventByFilter(tenantId.getId(), entityId, eventFilter, pageLink);
public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) {
if (eventFilter == null) {
eventDao.removeEvents(tenantId.getId(), entityId.getId(), startTime, endTime);
} else {
eventDao.removeEvents(tenantId.getId(), entityId.getId(), eventFilter, startTime, endTime);
}
}
@Override
public void removeEvents(TenantId tenantId, EntityId entityId) {
removeEvents(tenantId, entityId, null, null, null);
public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) {
eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs, cleanupDb);
}
@Override
public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) {
TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime);
PageData<Event> eventsPageData;
do {
if (eventFilter == null) {
eventsPageData = findEvents(tenantId, entityId, eventsPageLink);
} else {
eventsPageData = findEventsByFilter(tenantId, entityId, eventFilter, eventsPageLink);
}
public void migrateEvents() {
eventDao.migrateEvents(ttlInSec > 0 ? System.currentTimeMillis() - ttlInSec : 0, debugTtlInSec > 0 ? System.currentTimeMillis() - debugTtlInSec : 0);
}
eventDao.removeAllByIds(eventsPageData.getData().stream()
.map(IdBased::getUuidId)
.collect(Collectors.toList()));
} while (eventsPageData.hasNext());
private PageData<EventInfo> convert(EntityType entityType, PageData<? extends Event> pd) {
return new PageData<>(pd.getData() == null ? null :
pd.getData().stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList())
, pd.getTotalPages(), pd.getTotalElements(), pd.hasNext());
}
@Override
public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) {
eventDao.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs);
private List<EventInfo> convert(EntityType entityType, List<? extends Event> list) {
return list == null ? null : list.stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList());
}
}

67
dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java

@ -16,12 +16,11 @@
package org.thingsboard.server.dao.event;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventFilter;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.Dao;
import java.util.List;
import java.util.UUID;
@ -29,7 +28,7 @@ import java.util.UUID;
/**
* The Interface EventDao.
*/
public interface EventDao extends Dao<Event> {
public interface EventDao {
/**
* Save or update event object async
@ -39,27 +38,6 @@ public interface EventDao extends Dao<Event> {
*/
ListenableFuture<Void> saveAsync(Event event);
/**
* Find event by tenantId, entityId and eventUid.
*
* @param tenantId the tenantId
* @param entityId the entityId
* @param eventType the eventType
* @param eventUid the eventUid
* @return the event
*/
Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid);
/**
* Find events by tenantId, entityId and pageLink.
*
* @param tenantId the tenantId
* @param entityId the entityId
* @param pageLink the pageLink
* @return the event list
*/
PageData<Event> findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink);
/**
* Find events by tenantId, entityId, eventType and pageLink.
*
@ -69,9 +47,9 @@ public interface EventDao extends Dao<Event> {
* @param pageLink the pageLink
* @return the event list
*/
PageData<Event> findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink);
PageData<? extends Event> findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink);
PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink);
PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink);
/**
* Find latest events by tenantId, entityId and eventType.
@ -82,14 +60,37 @@ public interface EventDao extends Dao<Event> {
* @param limit the limit
* @return the event list
*/
List<Event> findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit);
List<? extends Event> findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit);
/**
* Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events.
* @param regularEventStartTs the start time of the interval to use to delete non debug events
* @param regularEventEndTs the end time of the interval to use to delete non debug events
* @param debugEventStartTs the start time of the interval to use to delete debug events
* @param debugEventEndTs the end time of the interval to use to delete debug events
* @param regularEventExpTs the expiration time of the regular events
* @param debugEventExpTs the expiration time of the debug events
* @param cleanupDb
*/
void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb);
/**
* Removes all events for the specified entity and time interval
*
* @param tenantId
* @param entityId
* @param startTime
* @param endTime
*/
void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs);
void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime);
/**
*
* Removes all events for the specified entity, event filter and time interval
*
* @param tenantId
* @param entityId
* @param eventFilter
* @param startTime
* @param endTime
*/
void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime);
void migrateEvents(long regularEventTs, long debugEventTs);
}

31
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@ -368,16 +368,35 @@ public class ModelConstants {
/**
* Cassandra event constants.
*/
public static final String EVENT_COLUMN_FAMILY_NAME = "event";
public static final String ERROR_EVENT_TABLE_NAME = "error_event";
public static final String LC_EVENT_TABLE_NAME = "lc_event";
public static final String STATS_EVENT_TABLE_NAME = "stats_event";
public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event";
public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event";
public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY;
public static final String EVENT_TYPE_PROPERTY = "event_type";
public static final String EVENT_UID_PROPERTY = "event_uid";
public static final String EVENT_ENTITY_TYPE_PROPERTY = ENTITY_TYPE_PROPERTY;
public static final String EVENT_SERVICE_ID_PROPERTY = "service_id";
public static final String EVENT_ENTITY_ID_PROPERTY = "entity_id";
public static final String EVENT_BODY_PROPERTY = "body";
public static final String EVENT_BY_TYPE_AND_ID_VIEW_NAME = "event_by_type_and_id";
public static final String EVENT_BY_ID_VIEW_NAME = "event_by_id";
public static final String EVENT_MESSAGES_PROCESSED_COLUMN_NAME = "e_messages_processed";
public static final String EVENT_ERRORS_OCCURRED_COLUMN_NAME = "e_errors_occurred";
public static final String EVENT_METHOD_COLUMN_NAME = "e_method";
public static final String EVENT_TYPE_COLUMN_NAME = "e_type";
public static final String EVENT_ERROR_COLUMN_NAME = "e_error";
public static final String EVENT_SUCCESS_COLUMN_NAME = "e_success";
public static final String EVENT_ENTITY_ID_COLUMN_NAME = "e_entity_id";
public static final String EVENT_ENTITY_TYPE_COLUMN_NAME = "e_entity_type";
public static final String EVENT_MSG_ID_COLUMN_NAME = "e_msg_id";
public static final String EVENT_MSG_TYPE_COLUMN_NAME = "e_msg_type";
public static final String EVENT_DATA_TYPE_COLUMN_NAME = "e_data_type";
public static final String EVENT_RELATION_TYPE_COLUMN_NAME = "e_relation_type";
public static final String EVENT_DATA_COLUMN_NAME = "e_data";
public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata";
public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message";
public static final String DEBUG_MODE = "debug_mode";

1
dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java

@ -31,7 +31,6 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME;

1
dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java

@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseSqlEntity;

64
dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java

@ -0,0 +1,64 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import static org.thingsboard.server.dao.model.ModelConstants.ERROR_EVENT_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METHOD_COLUMN_NAME;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = ERROR_EVENT_TABLE_NAME)
@NoArgsConstructor
public class ErrorEventEntity extends EventEntity<ErrorEvent> implements BaseEntity<ErrorEvent> {
@Column(name = EVENT_METHOD_COLUMN_NAME)
private String method;
@Column(name = EVENT_ERROR_COLUMN_NAME)
private String error;
public ErrorEventEntity(ErrorEvent event) {
super(event);
this.method = event.getMethod();
this.error = event.getError();
}
@Override
public ErrorEvent toData() {
return ErrorEvent.builder()
.tenantId(TenantId.fromUUID(tenantId))
.entityId(entityId)
.serviceId(serviceId)
.id(id)
.ts(ts)
.method(method)
.error(error)
.build();
}
}

116
dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java

@ -15,103 +15,77 @@
*/
package org.thingsboard.server.dao.model.sql;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.EventId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.dao.model.BaseEntity;
import org.thingsboard.server.dao.model.BaseSqlEntity;
import org.thingsboard.server.dao.util.mapping.JsonStringType;
import org.thingsboard.server.dao.model.ModelConstants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Table;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SERVICE_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@TypeDef(name = "json", typeClass = JsonStringType.class)
@Table(name = EVENT_COLUMN_FAMILY_NAME)
@NoArgsConstructor
public class EventEntity extends BaseSqlEntity<Event> implements BaseEntity<Event> {
@MappedSuperclass
public abstract class EventEntity<T extends Event> implements BaseEntity<T> {
@Column(name = EVENT_TENANT_ID_PROPERTY)
private UUID tenantId;
@Id
@Column(name = ModelConstants.ID_PROPERTY, columnDefinition = "uuid")
protected UUID id;
@Enumerated(EnumType.STRING)
@Column(name = EVENT_ENTITY_TYPE_PROPERTY)
private EntityType entityType;
@Column(name = EVENT_TENANT_ID_PROPERTY, columnDefinition = "uuid")
protected UUID tenantId;
@Column(name = EVENT_ENTITY_ID_PROPERTY)
private UUID entityId;
@Column(name = EVENT_ENTITY_ID_PROPERTY, columnDefinition = "uuid")
protected UUID entityId;
@Column(name = EVENT_TYPE_PROPERTY)
private String eventType;
@Column(name = EVENT_UID_PROPERTY)
private String eventUid;
@Type(type = "json")
@Column(name = EVENT_BODY_PROPERTY)
private JsonNode body;
@Column(name = EVENT_SERVICE_ID_PROPERTY)
protected String serviceId;
@Column(name = TS_COLUMN)
private long ts;
protected long ts;
public EventEntity(UUID id, UUID tenantId, UUID entityId, String serviceId, long ts) {
this.id = id;
this.tenantId = tenantId;
this.entityId = entityId;
this.serviceId = serviceId;
this.ts = ts;
}
public EventEntity(Event event) {
if (event.getId() != null) {
this.setUuid(event.getId().getId());
this.ts = getTs(event.getId().getId());
} else {
this.ts = System.currentTimeMillis();
}
this.setCreatedTime(event.getCreatedTime());
if (event.getTenantId() != null) {
this.tenantId = event.getTenantId().getId();
}
if (event.getEntityId() != null) {
this.entityType = event.getEntityId().getEntityType();
this.entityId = event.getEntityId().getId();
}
this.eventType = event.getType();
this.eventUid = event.getUid();
this.body = event.getBody();
this.id = event.getId().getId();
this.tenantId = event.getTenantId().getId();
this.entityId = event.getEntityId();
this.serviceId = event.getServiceId();
this.ts = event.getCreatedTime();
}
@Override
public UUID getUuid() {
return id;
}
@Override
public Event toData() {
Event event = new Event(new EventId(this.getUuid()));
event.setCreatedTime(createdTime);
event.setTenantId(TenantId.fromUUID(tenantId));
event.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId));
event.setBody(body);
event.setType(eventType);
event.setUid(eventUid);
return event;
public void setUuid(UUID id) {
this.id = id;
}
private static long getTs(UUID uuid) {
return (uuid.timestamp() - EPOCH_DIFF) / 10000;
@Override
public long getCreatedTime() {
return ts;
}
@Override
public void setCreatedTime(long createdTime) {
ts = createdTime;
}
}

69
dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java

@ -0,0 +1,69 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAME;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = LC_EVENT_TABLE_NAME)
@NoArgsConstructor
public class LifecycleEventEntity extends EventEntity<LifecycleEvent> implements BaseEntity<LifecycleEvent> {
@Column(name = EVENT_TYPE_COLUMN_NAME)
private String eventType;
@Column(name = EVENT_SUCCESS_COLUMN_NAME)
private boolean success;
@Column(name = EVENT_ERROR_COLUMN_NAME)
private String error;
public LifecycleEventEntity(LifecycleEvent event) {
super(event);
this.eventType = event.getLcEventType();
this.success = event.isSuccess();
this.error = event.getError();
}
@Override
public LifecycleEvent toData() {
return LifecycleEvent.builder()
.tenantId(TenantId.fromUUID(tenantId))
.entityId(entityId)
.serviceId(serviceId)
.id(id)
.ts(ts)
.lcEventType(eventType)
.success(success)
.error(error)
.build();
}
}

63
dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java

@ -0,0 +1,63 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RULE_CHAIN_DEBUG_EVENT_TABLE_NAME;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = RULE_CHAIN_DEBUG_EVENT_TABLE_NAME)
@NoArgsConstructor
public class RuleChainDebugEventEntity extends EventEntity<RuleChainDebugEvent> implements BaseEntity<RuleChainDebugEvent> {
@Column(name = EVENT_MESSAGE_COLUMN_NAME)
private String message;
@Column(name = EVENT_ERROR_COLUMN_NAME)
private String error;
public RuleChainDebugEventEntity(RuleChainDebugEvent event) {
super(event);
this.message = event.getMessage();
this.error = event.getError();
}
@Override
public RuleChainDebugEvent toData() {
return RuleChainDebugEvent.builder()
.tenantId(TenantId.fromUUID(tenantId))
.entityId(entityId)
.serviceId(serviceId)
.id(id)
.ts(ts)
.message(message)
.error(error).build();
}
}

109
dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java

@ -0,0 +1,109 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METADATA_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_RELATION_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RULE_NODE_DEBUG_EVENT_TABLE_NAME;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = RULE_NODE_DEBUG_EVENT_TABLE_NAME)
@NoArgsConstructor
public class RuleNodeDebugEventEntity extends EventEntity<RuleNodeDebugEvent> implements BaseEntity<RuleNodeDebugEvent> {
@Column(name = EVENT_TYPE_COLUMN_NAME)
private String eventType;
@Column(name = EVENT_ENTITY_ID_COLUMN_NAME)
private UUID eventEntityId;
@Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME)
private String eventEntityType;
@Column(name = EVENT_MSG_ID_COLUMN_NAME)
private UUID msgId;
@Column(name = EVENT_MSG_TYPE_COLUMN_NAME)
private String msgType;
@Column(name = EVENT_DATA_TYPE_COLUMN_NAME)
private String dataType;
@Column(name = EVENT_RELATION_TYPE_COLUMN_NAME)
private String relationType;
@Column(name = EVENT_DATA_COLUMN_NAME)
private String data;
@Column(name = EVENT_METADATA_COLUMN_NAME)
private String metadata;
@Column(name = EVENT_ERROR_COLUMN_NAME)
private String error;
public RuleNodeDebugEventEntity(RuleNodeDebugEvent event) {
super(event);
this.eventType = event.getEventType();
if (event.getEventEntity() != null) {
this.eventEntityId = event.getEventEntity().getId();
this.eventEntityType = event.getEventEntity().getEntityType().name();
}
this.msgId = event.getMsgId();
this.msgType = event.getMsgType();
this.dataType = event.getDataType();
this.relationType = event.getRelationType();
this.data = event.getData();
this.metadata = event.getMetadata();
this.error = event.getError();
}
@Override
public RuleNodeDebugEvent toData() {
var builder = RuleNodeDebugEvent.builder()
.tenantId(TenantId.fromUUID(tenantId))
.entityId(entityId)
.serviceId(serviceId)
.id(id)
.ts(ts)
.eventType(eventType)
.msgId(msgId)
.msgType(msgType)
.dataType(dataType)
.relationType(relationType)
.data(data)
.metadata(metadata)
.error(error);
if (eventEntityId != null) {
builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId));
}
return builder.build();
}
}

64
dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java

@ -0,0 +1,64 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.model.sql;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.model.BaseEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERRORS_OCCURRED_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGES_PROCESSED_COLUMN_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME;
@Data
@EqualsAndHashCode(callSuper = true)
@Entity
@Table(name = STATS_EVENT_TABLE_NAME)
@NoArgsConstructor
public class StatisticsEventEntity extends EventEntity<StatisticsEvent> implements BaseEntity<StatisticsEvent> {
@Column(name = EVENT_MESSAGES_PROCESSED_COLUMN_NAME)
private long messagesProcessed;
@Column(name = EVENT_ERRORS_OCCURRED_COLUMN_NAME)
private long errorsOccurred;
public StatisticsEventEntity(StatisticsEvent event) {
super(event);
this.messagesProcessed = event.getMessagesProcessed();
this.errorsOccurred = event.getErrorsOccurred();
}
@Override
public StatisticsEvent toData() {
return StatisticsEvent.builder()
.tenantId(TenantId.fromUUID(tenantId))
.entityId(entityId)
.serviceId(serviceId)
.id(id)
.ts(ts)
.messagesProcessed(messagesProcessed)
.errorsOccurred(errorsOccurred)
.build();
}
}

12
dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java

@ -17,7 +17,7 @@ package org.thingsboard.server.dao.service.validator;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
@ -27,14 +27,14 @@ public class EventDataValidator extends DataValidator<Event> {
@Override
protected void validateDataImpl(TenantId tenantId, Event event) {
if (event.getTenantId() == null) {
throw new DataValidationException("Tenant id should be specified!.");
}
if (event.getEntityId() == null) {
throw new DataValidationException("Entity id should be specified!.");
}
if (StringUtils.isEmpty(event.getType())) {
throw new DataValidationException("Event type should be specified!.");
}
if (event.getBody() == null) {
throw new DataValidationException("Event body should be specified!.");
if (StringUtils.isEmpty(event.getServiceId())) {
throw new DataValidationException("Service id should be specified!.");
}
}
}

114
dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java

@ -0,0 +1,114 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.dao.model.sql.ErrorEventEntity;
import org.thingsboard.server.dao.model.sql.LifecycleEventEntity;
import org.thingsboard.server.dao.model.sql.StatisticsEventEntity;
import java.util.List;
import java.util.UUID;
public interface ErrorEventRepository extends EventRepository<ErrorEventEntity, ErrorEvent>, JpaRepository<ErrorEventEntity, UUID> {
@Override
@Query(nativeQuery = true, value = "SELECT * FROM error_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit")
List<ErrorEventEntity> findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit);
@Override
@Query("SELECT e FROM ErrorEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
Page<ErrorEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT * FROM error_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
,
countQuery = "SELECT count(*) FROM error_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
Page<ErrorEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("method") String method,
@Param("error") String error,
Pageable pageable);
@Transactional
@Modifying
@Query("DELETE FROM ErrorEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime);
@Transactional
@Modifying
@Query(nativeQuery = true,
value = "DELETE FROM error_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("method") String method,
@Param("error") String error);
}

3
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java

@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.event;
public interface EventCleanupRepository {
void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs);
void cleanupEvents(long eventExpTime, boolean debug);
void migrateEvents(long regularEventTs, long debugEventTs);
}

199
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java

@ -24,12 +24,24 @@ import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionCallbackWithoutResult;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.server.dao.model.sql.EventEntity;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import javax.annotation.PostConstruct;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@Repository
@Transactional
@ -39,10 +51,7 @@ public class EventInsertRepository {
private static final String EMPTY_STR = "";
private static final String INSERT =
"INSERT INTO event (id, created_time, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " +
"ON CONFLICT DO NOTHING;";
private final Map<EventType, String> insertStmtMap = new ConcurrentHashMap<>();
@Autowired
protected JdbcTemplate jdbcTemplate;
@ -53,34 +62,172 @@ public class EventInsertRepository {
@Value("${sql.remove_null_chars:true}")
private boolean removeNullChars;
protected void save(List<EventEntity> entities) {
@PostConstruct
public void init() {
insertStmtMap.put(EventType.ERROR, "INSERT INTO " + EventType.ERROR.getTable() +
" (id, tenant_id, ts, entity_id, service_id, e_method, e_error) " +
"VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
insertStmtMap.put(EventType.LC_EVENT, "INSERT INTO " + EventType.LC_EVENT.getTable() +
" (id, tenant_id, ts, entity_id, service_id, e_type, e_success, e_error) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
insertStmtMap.put(EventType.STATS, "INSERT INTO " + EventType.STATS.getTable() +
" (id, tenant_id, ts, entity_id, service_id, e_messages_processed, e_errors_occurred) " +
"VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
insertStmtMap.put(EventType.DEBUG_RULE_NODE, "INSERT INTO " + EventType.DEBUG_RULE_NODE.getTable() +
" (id, tenant_id, ts, entity_id, service_id, e_type, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_data_type, e_relation_type, e_data, e_metadata, e_error) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() +
" (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " +
"VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;");
}
protected void save(List<Event> entities) {
Map<EventType, List<Event>> eventsByType = entities.stream().collect(Collectors.groupingBy(Event::getType, Collectors.toList()));
transactionTemplate.execute(new TransactionCallbackWithoutResult() {
@Override
protected void doInTransactionWithoutResult(TransactionStatus status) {
jdbcTemplate.batchUpdate(INSERT, new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
EventEntity event = entities.get(i);
ps.setObject(1, event.getId());
ps.setLong(2, event.getCreatedTime());
ps.setString(3, replaceNullChars(event.getBody().toString()));
ps.setObject(4, event.getEntityId());
ps.setString(5, event.getEntityType().name());
ps.setString(6, event.getEventType());
ps.setString(7, event.getEventUid());
ps.setObject(8, event.getTenantId());
ps.setLong(9, event.getTs());
}
@Override
public int getBatchSize() {
return entities.size();
}
});
for (var entry : eventsByType.entrySet()) {
jdbcTemplate.batchUpdate(insertStmtMap.get(entry.getKey()), getStatementSetter(entry.getKey(), entry.getValue()));
}
}
});
}
private BatchPreparedStatementSetter getStatementSetter(EventType eventType, List<Event> events) {
switch (eventType) {
case ERROR:
return getErrorEventSetter(events);
case LC_EVENT:
return getLcEventSetter(events);
case STATS:
return getStatsEventSetter(events);
case DEBUG_RULE_NODE:
return getRuleNodeEventSetter(events);
case DEBUG_RULE_CHAIN:
return getRuleChainEventSetter(events);
default:
throw new RuntimeException(eventType + " support is not implemented!");
}
}
private BatchPreparedStatementSetter getErrorEventSetter(List<Event> events) {
return new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
ErrorEvent event = (ErrorEvent) events.get(i);
setCommonEventFields(ps, event);
safePutString(ps, 6, event.getMethod());
safePutString(ps, 7, event.getError());
}
@Override
public int getBatchSize() {
return events.size();
}
};
}
private BatchPreparedStatementSetter getLcEventSetter(List<Event> events) {
return new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
LifecycleEvent event = (LifecycleEvent) events.get(i);
setCommonEventFields(ps, event);
safePutString(ps, 6, event.getLcEventType());
ps.setBoolean(7, event.isSuccess());
safePutString(ps, 8, event.getError());
}
@Override
public int getBatchSize() {
return events.size();
}
};
}
private BatchPreparedStatementSetter getStatsEventSetter(List<Event> events) {
return new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
StatisticsEvent event = (StatisticsEvent) events.get(i);
setCommonEventFields(ps, event);
ps.setLong(6, event.getMessagesProcessed());
ps.setLong(7, event.getErrorsOccurred());
}
@Override
public int getBatchSize() {
return events.size();
}
};
}
private BatchPreparedStatementSetter getRuleNodeEventSetter(List<Event> events) {
return new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
RuleNodeDebugEvent event = (RuleNodeDebugEvent) events.get(i);
setCommonEventFields(ps, event);
safePutString(ps, 6, event.getEventType());
safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null);
safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null);
safePutUUID(ps, 9, event.getMsgId());
safePutString(ps, 10, event.getMsgType());
safePutString(ps, 11, event.getDataType());
safePutString(ps, 12, event.getRelationType());
safePutString(ps, 13, event.getData());
safePutString(ps, 14, event.getMetadata());
safePutString(ps, 15, event.getError());
}
@Override
public int getBatchSize() {
return events.size();
}
};
}
private BatchPreparedStatementSetter getRuleChainEventSetter(List<Event> events) {
return new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
RuleChainDebugEvent event = (RuleChainDebugEvent) events.get(i);
setCommonEventFields(ps, event);
safePutString(ps, 6, event.getMessage());
safePutString(ps, 7, event.getError());
}
@Override
public int getBatchSize() {
return events.size();
}
};
}
void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException {
if (value != null) {
ps.setString(parameterIdx, replaceNullChars(value));
} else {
ps.setNull(parameterIdx, Types.VARCHAR);
}
}
void safePutUUID(PreparedStatement ps, int parameterIdx, UUID value) throws SQLException {
if (value != null) {
ps.setObject(parameterIdx, value);
} else {
ps.setNull(parameterIdx, Types.OTHER);
}
}
private void setCommonEventFields(PreparedStatement ps, Event event) throws SQLException {
ps.setObject(1, event.getId().getId());
ps.setObject(2, event.getTenantId().getId());
ps.setLong(3, event.getCreatedTime());
ps.setObject(4, event.getEntityId());
ps.setString(5, event.getServiceId());
}
private String replaceNullChars(String strValue) {
if (removeNullChars && strValue != null) {
return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR);

48
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java

@ -0,0 +1,48 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import lombok.Getter;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.event.EventType;
import javax.annotation.PostConstruct;
import java.util.concurrent.TimeUnit;
@Component
public class EventPartitionConfiguration {
@Getter
@Value("${sql.events.partition_size:168}")
private int regularPartitionSizeInHours;
@Getter
@Value("${sql.events.debug_partition_size:1}")
private int debugPartitionSizeInHours;
private long regularPartitionSizeInMs;
private long debugPartitionSizeInMs;
@PostConstruct
public void init() {
regularPartitionSizeInMs = TimeUnit.HOURS.toMillis(regularPartitionSizeInHours);
debugPartitionSizeInMs = TimeUnit.HOURS.toMillis(debugPartitionSizeInHours);
}
public long getPartitionSizeInMs(EventType eventType) {
return eventType.isDebug() ? debugPartitionSizeInMs : regularPartitionSizeInMs;
}
}

217
dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java

@ -21,224 +21,19 @@ import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.dao.model.sql.EventEntity;
import java.util.List;
import java.util.UUID;
/**
* Created by Valerii Sosliuk on 5/3/2017.
*/
public interface EventRepository extends JpaRepository<EventEntity, UUID> {
EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId,
EntityType entityType,
UUID entityId,
String eventType,
String eventUid);
EventEntity findByTenantIdAndEntityTypeAndEntityId(UUID tenantId,
EntityType entityType,
UUID entityId);
@Query("SELECT e FROM EventEntity e WHERE e.tenantId = :tenantId AND e.entityType = :entityType " +
"AND e.entityId = :entityId AND e.eventType = :eventType ORDER BY e.eventType DESC, e.id DESC")
List<EventEntity> findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType(
@Param("tenantId") UUID tenantId,
@Param("entityType") EntityType entityType,
@Param("entityId") UUID entityId,
@Param("eventType") String eventType,
Pageable pageable);
@Query("SELECT e FROM EventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityType = :entityType AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.createdTime >= :startTime) " +
"AND (:endTime IS NULL OR e.createdTime <= :endTime) " +
"AND LOWER(e.eventType) LIKE LOWER(CONCAT('%', :textSearch, '%'))"
)
Page<EventEntity> findEventsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId,
@Param("entityType") EntityType entityType,
@Param("entityId") UUID entityId,
@Param("textSearch") String textSearch,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query("SELECT e FROM EventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityType = :entityType AND e.entityId = :entityId " +
"AND e.eventType = :eventType " +
"AND (:startTime IS NULL OR e.createdTime >= :startTime) " +
"AND (:endTime IS NULL OR e.createdTime <= :endTime)"
)
Page<EventEntity> findEventsByTenantIdAndEntityIdAndEventType(@Param("tenantId") UUID tenantId,
@Param("entityType") EntityType entityType,
@Param("entityId") UUID entityId,
@Param("eventType") String eventType,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = :eventType " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " +
"AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " +
"AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " +
"AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " +
"AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " +
"AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " +
"AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " +
"AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%')) ",
countQuery = "SELECT count(*) FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = :eventType " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " +
"AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " +
"AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " +
"AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " +
"AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " +
"AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " +
"AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " +
"AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%'))"
)
Page<EventEntity> findDebugRuleNodeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("entityType") String entityType,
@Param("eventType") String eventType,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("type") String type,
@Param("server") String server,
@Param("entityName") String entityName,
@Param("relationType") String relationType,
@Param("bodyEntityId") String bodyEntityId,
@Param("msgType") String msgType,
@Param("isError") boolean isError,
@Param("error") String error,
@Param("data") String data,
@Param("metadata") String metadata,
Pageable pageable);
public interface EventRepository<T extends EventEntity<V>, V extends Event> {
@Query(nativeQuery = true,
value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'ERROR' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))",
countQuery = "SELECT count(*) FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'ERROR' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))")
Page<EventEntity> findErrorEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("entityType") String entityType,
@Param("startTime") Long startTime,
@Param("endTime") Long endTIme,
@Param("server") String server,
@Param("method") String method,
@Param("error") String error,
Pageable pageable);
List<T> findLatestEvents(UUID tenantId, UUID entityId, int limit);
@Query(nativeQuery = true,
value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'LC_EVENT' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " +
"AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))"
,
countQuery = "SELECT count(*) FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'LC_EVENT' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " +
"AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " +
"AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))"
)
Page<EventEntity> findLifeCycleEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("entityType") String entityType,
@Param("startTime") Long startTime,
@Param("endTime") Long endTIme,
@Param("server") String server,
@Param("event") String event,
@Param("statusFilterEnabled") boolean statusFilterEnabled,
@Param("statusFilter") boolean statusFilter,
@Param("error") String error,
Pageable pageable);
Page<T> findEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime, Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'STATS' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " +
"AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ",
countQuery = "SELECT count(*) FROM " +
"(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_type = :entityType " +
"AND e.entity_id = :entityId " +
"AND e.event_type = 'LC_EVENT' " +
"AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " +
") AS e WHERE " +
"(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " +
"AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " +
"AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ")
Page<EventEntity> findStatisticsEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("entityType") String entityType,
@Param("startTime") Long startTime,
@Param("endTime") Long endTIme,
@Param("server") String server,
@Param("messagesProcessed") Integer messagesProcessed,
@Param("errorsOccurred") Integer errorsOccurred,
Pageable pageable);
void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime);
}

402
dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java

@ -18,19 +18,20 @@ package org.thingsboard.server.dao.sql.event;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.dao.DataIntegrityViolationException;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.event.DebugEvent;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.event.RuleChainDebugEventFilter;
import org.thingsboard.server.common.data.event.RuleNodeDebugEventFilter;
import org.thingsboard.server.common.data.event.ErrorEventFilter;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventFilter;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.event.LifeCycleEventFilter;
import org.thingsboard.server.common.data.event.StatisticsEventFilter;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EventId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
@ -38,32 +39,47 @@ import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.event.EventDao;
import org.thingsboard.server.dao.model.sql.EventEntity;
import org.thingsboard.server.dao.sql.JpaAbstractDao;
import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams;
import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper;
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository;
import org.thingsboard.server.dao.timeseries.SqlPartition;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
/**
* Created by Valerii Sosliuk on 5/3/2017.
*/
@Slf4j
@Component
public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implements EventDao {
public class JpaBaseEventDao implements EventDao {
private final Map<EventType, Map<Long, SqlPartition>> partitionsByEventType = new ConcurrentHashMap<>();
private static final ReentrantLock partitionCreationLock = new ReentrantLock();
@Autowired
private EventPartitionConfiguration partitionConfiguration;
@Autowired
private SqlPartitioningRepository partitioningRepository;
private final UUID systemTenantId = NULL_UUID;
@Autowired
private LifecycleEventRepository lcEventRepository;
@Autowired
private StatisticsEventRepository statsEventRepository;
@Autowired
private EventRepository eventRepository;
private ErrorEventRepository errorEventRepository;
@Autowired
private EventInsertRepository eventInsertRepository;
@ -71,15 +87,11 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
@Autowired
private EventCleanupRepository eventCleanupRepository;
@Override
protected Class<EventEntity> getEntityClass() {
return EventEntity.class;
}
@Autowired
private RuleNodeDebugEventRepository ruleNodeDebugEventRepository;
@Override
protected JpaRepository<EventEntity, UUID> getRepository() {
return eventRepository;
}
@Autowired
private RuleChainDebugEventRepository ruleChainDebugEventRepository;
@Autowired
ScheduledLogExecutorComponent logExecutor;
@ -102,10 +114,15 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
@Value("${sql.batch_sort:false}")
private boolean batchSortEnabled;
private TbSqlBlockingQueueWrapper<EventEntity> queue;
private TbSqlBlockingQueueWrapper<Event> queue;
private final Map<EventType, EventRepository<?, ?>> repositories = new ConcurrentHashMap<>();
@PostConstruct
private void init() {
for (EventType eventType : EventType.values()) {
partitionsByEventType.put(eventType, new ConcurrentHashMap<>());
}
TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder()
.logName("Events")
.batchSize(batchSize)
@ -114,11 +131,14 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
.statsNamePrefix("events")
.batchSortEnabled(batchSortEnabled)
.build();
Function<EventEntity, Integer> hashcodeFunction = entity -> entity.getEntityId().hashCode();
Function<Event, Integer> hashcodeFunction = entity -> Objects.hash(super.hashCode(), entity.getTenantId(), entity.getEntityId());
queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory);
queue.init(logExecutor, v -> eventInsertRepository.save(v),
Comparator.comparing((EventEntity eventEntity) -> eventEntity.getTs())
);
queue.init(logExecutor, v -> eventInsertRepository.save(v), Comparator.comparing(Event::getCreatedTime));
repositories.put(EventType.LC_EVENT, lcEventRepository);
repositories.put(EventType.STATS, statsEventRepository);
repositories.put(EventType.ERROR, errorEventRepository);
repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository);
repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository);
}
@PreDestroy
@ -143,117 +163,153 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
event.setCreatedTime(System.currentTimeMillis());
}
}
if (StringUtils.isEmpty(event.getUid())) {
event.setUid(event.getId().toString());
}
return save(new EventEntity(event));
savePartitionIfNotExist(event);
return queue.add(event);
}
private ListenableFuture<Void> save(EventEntity entity) {
log.debug("Save event [{}] ", entity);
if (entity.getTenantId() == null) {
log.trace("Save system event with predefined id {}", systemTenantId);
entity.setTenantId(systemTenantId);
}
if (entity.getUuid() == null) {
entity.setUuid(Uuids.timeBased());
private void savePartitionIfNotExist(Event event) {
EventType type = event.getType();
var partitionsMap = partitionsByEventType.get(type);
var partitionDuration = partitionConfiguration.getPartitionSizeInMs(type);
long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % partitionDuration);
if (partitionsMap.get(partitionStartTs) == null) {
savePartition(partitionsMap, new SqlPartition(type.getTable(), partitionStartTs, partitionStartTs + partitionDuration, Long.toString(partitionStartTs)));
}
if (StringUtils.isEmpty(entity.getEventUid())) {
entity.setEventUid(entity.getUuid().toString());
}
return addToQueue(entity);
}
private ListenableFuture<Void> addToQueue(EventEntity entity) {
return queue.add(entity);
private void savePartition(Map<Long, SqlPartition> partitionsMap, SqlPartition sqlPartition) {
if (!partitionsMap.containsKey(sqlPartition.getStart())) {
partitionCreationLock.lock();
try {
log.trace("Saving partition: {}", sqlPartition);
partitioningRepository.save(sqlPartition);
log.trace("Adding partition to map: {}", sqlPartition);
partitionsMap.put(sqlPartition.getStart(), sqlPartition);
} catch (DataIntegrityViolationException ex) {
log.trace("Error occurred during partition save:", ex);
if (ex.getCause() instanceof ConstraintViolationException) {
log.warn("Saving partition [{}] rejected. Event data will save to the DEFAULT partition.", sqlPartition.getPartitionDate());
partitionsMap.put(sqlPartition.getStart(), sqlPartition);
} else {
throw new RuntimeException(ex);
}
} finally {
partitionCreationLock.unlock();
}
}
}
@Override
public Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid) {
return DaoUtil.getData(eventRepository.findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(
tenantId, entityId.getEntityType(), entityId.getId(), eventType, eventUid));
public PageData<? extends Event> findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink) {
return DaoUtil.toPageData(getEventRepository(eventType).findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink)));
}
@Override
public PageData<Event> findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) {
return DaoUtil.toPageData(
eventRepository
.findEventsByTenantIdAndEntityId(
tenantId,
entityId.getEntityType(),
entityId.getId(),
Objects.toString(pageLink.getTextSearch(), ""),
pageLink.getStartTime(),
pageLink.getEndTime(),
DaoUtil.toPageable(pageLink)));
public PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink) {
if (eventFilter.isNotEmpty()) {
switch (eventFilter.getEventType()) {
case DEBUG_RULE_NODE:
return findEventByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, pageLink);
case DEBUG_RULE_CHAIN:
return findEventByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, pageLink);
case LC_EVENT:
return findEventByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, pageLink);
case ERROR:
return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink);
case STATS:
return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink);
default:
throw new RuntimeException("Not supported event type: " + eventFilter.getEventType());
}
} else {
return findEvents(tenantId, entityId, eventFilter.getEventType(), pageLink);
}
}
@Override
public PageData<Event> findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink) {
return DaoUtil.toPageData(
eventRepository
.findEventsByTenantIdAndEntityIdAndEventType(
tenantId,
entityId.getEntityType(),
entityId.getId(),
eventType,
pageLink.getStartTime(),
pageLink.getEndTime(),
DaoUtil.toPageable(pageLink)));
public void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime) {
log.debug("[{}][{}] Remove events [{}-{}] ", tenantId, entityId, startTime, endTime);
for (EventType eventType : EventType.values()) {
getEventRepository(eventType).removeEvents(tenantId, entityId, startTime, endTime);
}
}
@Override
public PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) {
if (eventFilter.hasFilterForJsonBody()) {
public void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime) {
if (eventFilter.isNotEmpty()) {
switch (eventFilter.getEventType()) {
case DEBUG_RULE_NODE:
removeEventsByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, startTime, endTime);
break;
case DEBUG_RULE_CHAIN:
return findEventByFilter(tenantId, entityId, (DebugEvent) eventFilter, pageLink);
removeEventsByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, startTime, endTime);
break;
case LC_EVENT:
return findEventByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, pageLink);
removeEventsByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, startTime, endTime);
break;
case ERROR:
return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink);
removeEventsByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, startTime, endTime);
break;
case STATS:
return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink);
removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime);
break;
default:
throw new RuntimeException("Not supported event type: " + eventFilter.getEventType());
}
} else {
return findEvents(tenantId, entityId, eventFilter.getEventType().name(), pageLink);
getEventRepository(eventFilter.getEventType()).removeEvents(tenantId, entityId, startTime, endTime);
}
}
private PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, DebugEvent eventFilter, TimePageLink pageLink) {
@Override
public void migrateEvents(long regularEventTs, long debugEventTs) {
eventCleanupRepository.migrateEvents(regularEventTs, debugEventTs);
}
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, TimePageLink pageLink) {
return DaoUtil.toPageData(
eventRepository.findDebugRuleNodeEvents(
ruleChainDebugEventRepository.findEvents(
tenantId,
entityId.getId(),
entityId.getEntityType().name(),
eventFilter.getEventType().name(),
notNull(pageLink.getStartTime()),
notNull(pageLink.getEndTime()),
eventFilter.getMsgDirectionType(),
entityId,
pageLink.getStartTime(),
pageLink.getEndTime(),
eventFilter.getServer(),
eventFilter.getEntityName(),
eventFilter.getRelationType(),
eventFilter.getEntityId(),
eventFilter.getMsgType(),
eventFilter.getMessage(),
eventFilter.isError(),
eventFilter.getErrorStr(),
DaoUtil.toPageable(pageLink)));
}
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, TimePageLink pageLink) {
parseUUID(eventFilter.getEntityId(), "Entity Id");
parseUUID(eventFilter.getMsgId(), "Message Id");
return DaoUtil.toPageData(
ruleNodeDebugEventRepository.findEvents(
tenantId,
entityId,
pageLink.getStartTime(),
pageLink.getEndTime(),
eventFilter.getServer(),
eventFilter.getMsgDirectionType(),
eventFilter.getEntityId(),
eventFilter.getEntityType(),
eventFilter.getMsgId(),
eventFilter.getMsgType(),
eventFilter.getRelationType(),
eventFilter.getDataSearch(),
eventFilter.getMetadataSearch(),
eventFilter.isError(),
eventFilter.getErrorStr(),
DaoUtil.toPageable(pageLink)));
}
private PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) {
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) {
return DaoUtil.toPageData(
eventRepository.findErrorEvents(
errorEventRepository.findEvents(
tenantId,
entityId.getId(),
entityId.getEntityType().name(),
notNull(pageLink.getStartTime()),
notNull(pageLink.getEndTime()),
entityId,
pageLink.getStartTime(),
pageLink.getEndTime(),
eventFilter.getServer(),
eventFilter.getMethod(),
eventFilter.getErrorStr(),
@ -261,16 +317,15 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
);
}
private PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) {
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) {
boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus());
boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success");
return DaoUtil.toPageData(
eventRepository.findLifeCycleEvents(
lcEventRepository.findEvents(
tenantId,
entityId.getId(),
entityId.getEntityType().name(),
notNull(pageLink.getStartTime()),
notNull(pageLink.getEndTime()),
entityId,
pageLink.getStartTime(),
pageLink.getEndTime(),
eventFilter.getServer(),
eventFilter.getEvent(),
statusFilterEnabled,
@ -280,44 +335,145 @@ public class JpaBaseEventDao extends JpaAbstractDao<EventEntity, Event> implemen
);
}
private PageData<Event> findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) {
private PageData<? extends Event> findEventByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) {
return DaoUtil.toPageData(
eventRepository.findStatisticsEvents(
statsEventRepository.findEvents(
tenantId,
entityId.getId(),
entityId.getEntityType().name(),
notNull(pageLink.getStartTime()),
notNull(pageLink.getEndTime()),
entityId,
pageLink.getStartTime(),
pageLink.getEndTime(),
eventFilter.getServer(),
notNull(eventFilter.getMessagesProcessed()),
notNull(eventFilter.getErrorsOccurred()),
eventFilter.getMinMessagesProcessed(),
eventFilter.getMaxMessagesProcessed(),
eventFilter.getMinErrorsOccurred(),
eventFilter.getMaxErrorsOccurred(),
DaoUtil.toPageable(pageLink))
);
}
@Override
public List<Event> findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit) {
List<EventEntity> latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType(
private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) {
ruleChainDebugEventRepository.removeEvents(
tenantId,
entityId,
startTime,
endTime,
eventFilter.getServer(),
eventFilter.getMessage(),
eventFilter.isError(),
eventFilter.getErrorStr());
}
private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, Long startTime, Long endTime) {
parseUUID(eventFilter.getEntityId(), "Entity Id");
parseUUID(eventFilter.getMsgId(), "Message Id");
ruleNodeDebugEventRepository.removeEvents(
tenantId,
entityId,
startTime,
endTime,
eventFilter.getServer(),
eventFilter.getMsgDirectionType(),
eventFilter.getEntityId(),
eventFilter.getEntityType(),
eventFilter.getMsgId(),
eventFilter.getMsgType(),
eventFilter.getRelationType(),
eventFilter.getDataSearch(),
eventFilter.getMetadataSearch(),
eventFilter.isError(),
eventFilter.getErrorStr());
}
private void removeEventsByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, Long startTime, Long endTime) {
errorEventRepository.removeEvents(
tenantId,
entityId.getEntityType(),
entityId.getId(),
eventType,
PageRequest.of(0, limit));
return DaoUtil.convertDataList(latest);
entityId,
startTime,
endTime,
eventFilter.getServer(),
eventFilter.getMethod(),
eventFilter.getErrorStr());
}
private void removeEventsByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, Long startTime, Long endTime) {
boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus());
boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success");
lcEventRepository.removeEvents(
tenantId,
entityId,
startTime,
endTime,
eventFilter.getServer(),
eventFilter.getEvent(),
statusFilterEnabled,
statusFilter,
eventFilter.getErrorStr());
}
private void removeEventsByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, Long startTime, Long endTime) {
statsEventRepository.removeEvents(
tenantId,
entityId,
startTime,
endTime,
eventFilter.getServer(),
eventFilter.getMinMessagesProcessed(),
eventFilter.getMaxMessagesProcessed(),
eventFilter.getMinErrorsOccurred(),
eventFilter.getMaxErrorsOccurred()
);
}
@Override
public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) {
log.info("Going to cleanup old events. Interval for regular events: [{}:{}], for debug events: [{}:{}]", regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs);
eventCleanupRepository.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs);
public List<? extends Event> findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) {
return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit));
}
private long notNull(Long value) {
return value != null ? value : 0;
@Override
public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) {
if (regularEventExpTs > 0) {
log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs);
if (cleanupDb) {
eventCleanupRepository.cleanupEvents(regularEventExpTs, false);
}
cleanupPartitions(regularEventExpTs, false);
}
if (debugEventExpTs > 0) {
log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs);
if (cleanupDb) {
eventCleanupRepository.cleanupEvents(debugEventExpTs, true);
}
cleanupPartitions(debugEventExpTs, true);
}
}
private int notNull(Integer value) {
return value != null ? value : 0;
private void cleanupPartitions(long expTime, boolean isDebug) {
for (EventType eventType : EventType.values()) {
if (eventType.isDebug() == isDebug) {
Map<Long, SqlPartition> partitions = partitionsByEventType.get(eventType);
partitions.keySet().removeIf(startTs -> startTs + partitionConfiguration.getPartitionSizeInMs(eventType) < expTime);
}
}
}
private void parseUUID(String src, String paramName) {
if (!StringUtils.isEmpty(src)) {
try {
UUID.fromString(src);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("Failed to convert " + paramName + " to UUID!");
}
}
}
private EventRepository<? extends EventEntity<?>, ?> getEventRepository(EventType eventType) {
var repository = repositories.get(eventType);
if (repository == null) {
throw new RuntimeException("Event type: " + eventType + " is not supported!");
}
return repository;
}
}

118
dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java

@ -0,0 +1,118 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.event.LifecycleEvent;
import org.thingsboard.server.dao.model.sql.LifecycleEventEntity;
import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity;
import java.util.List;
import java.util.UUID;
public interface LifecycleEventRepository extends EventRepository<LifecycleEventEntity, LifecycleEvent>, JpaRepository<LifecycleEventEntity, UUID> {
@Override
@Query(nativeQuery = true, value = "SELECT * FROM lc_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit")
List<LifecycleEventEntity> findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit);
@Query("SELECT e FROM LifecycleEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
Page<LifecycleEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT * FROM lc_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
,
countQuery = "SELECT count(*) FROM lc_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
Page<LifecycleEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("eventType") String eventType,
@Param("statusFilterEnabled") boolean statusFilterEnabled,
@Param("statusFilter") boolean statusFilter,
@Param("error") String error,
Pageable pageable);
@Transactional
@Modifying
@Query("DELETE FROM LifecycleEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime);
@Transactional
@Modifying
@Query(nativeQuery = true,
value = "DELETE FROM lc_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("eventType") String eventType,
@Param("statusFilterEnabled") boolean statusFilterEnabled,
@Param("statusFilter") boolean statusFilter,
@Param("error") String error);
}

117
dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java

@ -0,0 +1,117 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.event.RuleChainDebugEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity;
import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity;
import java.util.List;
import java.util.UUID;
public interface RuleChainDebugEventRepository extends EventRepository<RuleChainDebugEventEntity, RuleChainDebugEvent>, JpaRepository<RuleChainDebugEventEntity, UUID> {
@Override
@Query(nativeQuery = true, value = "SELECT * FROM rule_chain_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit")
List<RuleChainDebugEventEntity> findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit);
@Override
@Query("SELECT e FROM RuleChainDebugEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
Page<RuleChainDebugEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT * FROM rule_chain_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
,
countQuery = "SELECT count(*) FROM rule_chain_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
Page<RuleChainDebugEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("message") String message,
@Param("isError") boolean isError,
@Param("error") String error,
Pageable pageable);
@Transactional
@Modifying
@Query("DELETE FROM RuleChainDebugEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime);
@Transactional
@Modifying
@Query(nativeQuery = true,
value = "DELETE FROM rule_chain_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))")
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("message") String message,
@Param("isError") boolean isError,
@Param("error") String error);
}

153
dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java

@ -0,0 +1,153 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.event.ErrorEvent;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.dao.model.sql.ErrorEventEntity;
import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity;
import org.thingsboard.server.dao.model.sql.StatisticsEventEntity;
import java.util.List;
import java.util.UUID;
public interface RuleNodeDebugEventRepository extends EventRepository<RuleNodeDebugEventEntity, RuleNodeDebugEvent>, JpaRepository<RuleNodeDebugEventEntity, UUID> {
@Override
@Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit")
List<RuleNodeDebugEventEntity> findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit);
@Override
@Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
Page<RuleNodeDebugEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT * FROM rule_node_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " +
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " +
"AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " +
"AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
,
countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " +
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " +
"AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " +
"AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))"
)
Page<RuleNodeDebugEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("eventType") String type,
@Param("eventEntityId") String eventEntityId,
@Param("eventEntityType") String eventEntityType,
@Param("msgId") String eventMsgId,
@Param("msgType") String eventMsgType,
@Param("relationType") String relationType,
@Param("data") String data,
@Param("metadata") String metadata,
@Param("isError") boolean isError,
@Param("error") String error,
Pageable pageable);
@Transactional
@Modifying
@Query("DELETE FROM RuleNodeDebugEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime);
@Transactional
@Modifying
@Query(nativeQuery = true,
value = "DELETE FROM rule_node_debug_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " +
"AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " +
"AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " +
"AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " +
"AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " +
"AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " +
"AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " +
"AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " +
"AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " +
"AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))")
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("eventType") String type,
@Param("eventEntityId") String eventEntityId,
@Param("eventEntityType") String eventEntityType,
@Param("msgId") String eventMsgId,
@Param("msgType") String eventMsgType,
@Param("relationType") String relationType,
@Param("data") String data,
@Param("metadata") String metadata,
@Param("isError") boolean isError,
@Param("error") String error);
}

161
dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java

@ -16,38 +16,175 @@
package org.thingsboard.server.dao.sql.event;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Repository;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Slf4j
@Repository
public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository {
private static final String SELECT_PARTITIONS_STMT = "SELECT tablename from pg_tables WHERE schemaname = 'public' and tablename like concat(?, '_%')";
private static final int PSQL_VERSION_14 = 140000;
@Autowired
private EventPartitionConfiguration partitionConfiguration;
private volatile Integer currentServerVersion;
@Override
public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) {
public void cleanupEvents(long eventExpTime, boolean debug) {
for (EventType eventType : EventType.values()) {
if (eventType.isDebug() == debug) {
cleanupEvents(eventType, eventExpTime);
}
}
}
@Override
public void migrateEvents(long regularEventTs, long debugEventTs) {
regularEventTs = Math.max(regularEventTs, 1480982400000L);
debugEventTs = Math.max(debugEventTs, 1480982400000L);
callMigrateFunctionByPartitions("regular", "migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours());
callMigrateFunctionByPartitions("debug", "migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours());
try (Connection connection = dataSource.getConnection();
PreparedStatement stmt = connection.prepareStatement("call cleanup_events_by_ttl(?,?,?,?,?)")) {
stmt.setLong(1, regularEventStartTs);
stmt.setLong(2, regularEventEndTs);
stmt.setLong(3, debugEventStartTs);
stmt.setLong(4, debugEventEndTs);
stmt.setLong(5, 0);
stmt.setQueryTimeout((int) TimeUnit.HOURS.toSeconds(1));
PreparedStatement dropFunction1 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_regular_events(bigint, bigint, int)");
PreparedStatement dropFunction2 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_debug_events(bigint, bigint, int)");
PreparedStatement dropTable = connection.prepareStatement("DROP TABLE IF EXISTS event")) {
dropFunction1.execute();
dropFunction2.execute();
dropTable.execute();
} catch (SQLException e) {
log.error("SQLException occurred during drop of the `events` table", e);
throw new RuntimeException(e);
}
}
private void callMigrateFunctionByPartitions(String logTag, String functionName, long startTs, int partitionSizeInHours) {
long currentTs = System.currentTimeMillis();
var regularPartitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours);
long numberOfPartitions = (currentTs - startTs) / regularPartitionStepInMs;
if (numberOfPartitions > 1000) {
log.error("Please adjust your {} events partitioning configuration. " +
"Configuration with partition size of {} hours and corresponding TTL will use {} (>1000) partitions which is not recommended!",
logTag, partitionSizeInHours, numberOfPartitions);
throw new RuntimeException("Please adjust your " + logTag + " events partitioning configuration. " +
"Configuration with partition size of " + partitionSizeInHours + " hours and corresponding TTL will use " +
+numberOfPartitions + " (>1000) partitions which is not recommended!");
}
while (startTs < currentTs) {
var endTs = startTs + regularPartitionStepInMs;
log.info("Migrate {} events for time period: [{},{}]", logTag, startTs, endTs);
callMigrateFunction(functionName, startTs, startTs + regularPartitionStepInMs, partitionSizeInHours);
startTs = endTs;
}
log.info("Migrate {} events done.", logTag);
}
private void callMigrateFunction(String functionName, long startTs, long endTs, int partitionSizeInHours) {
try (Connection connection = dataSource.getConnection();
PreparedStatement stmt = connection.prepareStatement("call " + functionName + "(?,?,?)")) {
stmt.setLong(1, startTs);
stmt.setLong(2, endTs);
stmt.setInt(3, partitionSizeInHours);
stmt.execute();
printWarnings(stmt);
try (ResultSet resultSet = stmt.getResultSet()){
resultSet.next();
log.info("Total events removed by TTL: [{}]", resultSet.getLong(1));
} catch (SQLException e) {
if (e.getMessage() == null || !e.getMessage().contains("relation \"event\" does not exist")) {
log.error("[{}] SQLException occurred during execution of {} with parameters {} and {}", functionName, startTs, partitionSizeInHours, e);
throw new RuntimeException(e);
}
}
}
private void cleanupEvents(EventType eventType, long eventExpTime) {
var partitionDuration = partitionConfiguration.getPartitionSizeInMs(eventType);
List<Long> partitions = fetchPartitions(eventType);
for (var partitionTs : partitions) {
var partitionEndTs = partitionTs + partitionDuration;
if (partitionEndTs < eventExpTime) {
log.info("[{}] Detaching expired partition: [{}-{}]", eventType, partitionTs, partitionEndTs);
if (detachAndDropPartition(eventType, partitionTs)) {
log.info("[{}] Detached expired partition: {}", eventType, partitionTs);
}
} else {
log.debug("[{}] Skip valid partition: {}", eventType, partitionTs);
}
}
}
private List<Long> fetchPartitions(EventType eventType) {
List<Long> partitions = new ArrayList<>();
try (Connection connection = dataSource.getConnection();
PreparedStatement stmt = connection.prepareStatement(SELECT_PARTITIONS_STMT)) {
stmt.setString(1, eventType.getTable());
stmt.execute();
try (ResultSet resultSet = stmt.getResultSet()) {
while (resultSet.next()) {
String partitionTableName = resultSet.getString(1);
String partitionTsStr = partitionTableName.substring(eventType.getTable().length() + 1);
try {
partitions.add(Long.parseLong(partitionTsStr));
} catch (NumberFormatException nfe) {
log.warn("Failed to parse table name: {}", partitionTableName);
}
}
}
} catch (SQLException e) {
log.error("SQLException occurred during events TTL task execution ", e);
}
return partitions;
}
private boolean detachAndDropPartition(EventType eventType, long partitionTs) {
String tablePartition = eventType.getTable() + "_" + partitionTs;
String detachPsqlStmtStr = "ALTER TABLE " + eventType.getTable() + " DETACH PARTITION " + tablePartition;
if (getCurrentServerVersion() >= PSQL_VERSION_14) {
detachPsqlStmtStr += " CONCURRENTLY";
}
String dropStmtStr = "DROP TABLE " + tablePartition;
try (Connection connection = dataSource.getConnection();
PreparedStatement detachStmt = connection.prepareStatement(detachPsqlStmtStr);
PreparedStatement dropStmt = connection.prepareStatement(dropStmtStr)) {
detachStmt.execute();
dropStmt.execute();
return true;
} catch (SQLException e) {
log.error("[{}] SQLException occurred during detach and drop of the partition: {}", eventType, partitionTs, e);
}
return false;
}
private synchronized int getCurrentServerVersion() {
if (currentServerVersion == null) {
try (Connection connection = dataSource.getConnection();
PreparedStatement versionStmt = connection.prepareStatement("SELECT current_setting('server_version_num')")) {
versionStmt.execute();
try (ResultSet resultSet = versionStmt.getResultSet()) {
while (resultSet.next()) {
currentServerVersion = resultSet.getInt(1);
}
}
} catch (SQLException e) {
log.warn("SQLException occurred during fetch of the server version", e);
}
if (currentServerVersion == null) {
currentServerVersion = 0;
}
}
return currentServerVersion;
}
}

120
dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java

@ -0,0 +1,120 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sql.event;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.dao.model.sql.StatisticsEventEntity;
import java.util.List;
import java.util.UUID;
public interface StatisticsEventRepository extends EventRepository<StatisticsEventEntity, StatisticsEvent>, JpaRepository<StatisticsEventEntity, UUID> {
@Override
@Query(nativeQuery = true, value = "SELECT * FROM stats_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit")
List<StatisticsEventEntity> findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit);
@Query("SELECT e FROM StatisticsEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
Page<StatisticsEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
Pageable pageable);
@Query(nativeQuery = true,
value = "SELECT * FROM stats_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " +
"AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " +
"AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " +
"AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)"
,
countQuery = "SELECT count(*) FROM stats_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " +
"AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " +
"AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " +
"AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)"
)
Page<StatisticsEventEntity> findEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("minMessagesProcessed") Integer minMessagesProcessed,
@Param("maxMessagesProcessed") Integer maxMessagesProcessed,
@Param("minErrorsOccurred") Integer minErrorsOccurred,
@Param("maxErrorsOccurred") Integer maxErrorsOccurred,
Pageable pageable);
@Transactional
@Modifying
@Query("DELETE FROM StatisticsEventEntity e WHERE " +
"e.tenantId = :tenantId " +
"AND e.entityId = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime);
@Transactional
@Modifying
@Query(nativeQuery = true,
value = "DELETE FROM stats_event e WHERE " +
"e.tenant_id = :tenantId " +
"AND e.entity_id = :entityId " +
"AND (:startTime IS NULL OR e.ts >= :startTime) " +
"AND (:endTime IS NULL OR e.ts <= :endTime) " +
"AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " +
"AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " +
"AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " +
"AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " +
"AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)"
)
void removeEvents(@Param("tenantId") UUID tenantId,
@Param("entityId") UUID entityId,
@Param("startTime") Long startTime,
@Param("endTime") Long endTime,
@Param("serviceId") String server,
@Param("minMessagesProcessed") Integer minMessagesProcessed,
@Param("maxMessagesProcessed") Integer maxMessagesProcessed,
@Param("minErrorsOccurred") Integer minErrorsOccurred,
@Param("maxErrorsOccurred") Integer maxErrorsOccurred);
}

4
dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java

@ -23,7 +23,6 @@ import org.thingsboard.server.dao.util.SqlTsDao;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
@SqlTsDao
@Repository
@Transactional
public class SqlPartitioningRepository {
@ -32,8 +31,7 @@ public class SqlPartitioningRepository {
private EntityManager entityManager;
public void save(SqlPartition partition) {
entityManager.createNativeQuery(partition.getQuery())
.executeUpdate();
entityManager.createNativeQuery(partition.getQuery()).executeUpdate();
}
}

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java

@ -132,7 +132,7 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao
long partitionEndTs = toMills(localDateTimeEnd);
ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC);
String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern()));
savePartition(new SqlPartition(partitionStartTs, partitionEndTs, partitionDate));
savePartition(new SqlPartition(SqlPartition.TS_KV, partitionStartTs, partitionEndTs, partitionDate));
}
}
}

10
dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java

@ -20,21 +20,21 @@ import lombok.Data;
@Data
public class SqlPartition {
private static final String TABLE_REGEX = "ts_kv_";
public static final String TS_KV = "ts_kv";
private long start;
private long end;
private String partitionDate;
private String query;
public SqlPartition(long start, long end, String partitionDate) {
public SqlPartition(String table, long start, long end, String partitionDate) {
this.start = start;
this.end = end;
this.partitionDate = partitionDate;
this.query = createStatement(start, end, partitionDate);
this.query = createStatement(table, start, end, partitionDate);
}
private String createStatement(long start, long end, String partitionDate) {
return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv FOR VALUES FROM (" + start + ") TO (" + end + ")";
private String createStatement(String table, long start, long end, String partitionDate) {
return "CREATE TABLE IF NOT EXISTS " + table + "_" + partitionDate + " PARTITION OF " + table + " FOR VALUES FROM (" + start + ") TO (" + end + ")";
}
}

24
dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql

@ -21,18 +21,18 @@
-- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts
-- That why we need to define DESC index explicitly as (ts DESC NULLS LAST)
CREATE INDEX IF NOT EXISTS idx_event_ts
ON public.event
(ts DESC NULLS LAST)
WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main
ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
COMMENT ON INDEX public.idx_event_ts
IS 'This index helps to delete events by TTL using timestamp';
CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main
ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des
ON public.event
(tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST)
WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_stats_event_main
ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_lc_event_main
ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
CREATE INDEX IF NOT EXISTS idx_error_event_main
ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95);
COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des
IS 'This index helps to open latest events on UI fast';

68
dao/src/main/resources/sql/schema-entities.sql

@ -323,18 +323,64 @@ CREATE TABLE IF NOT EXISTS device_credentials (
CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id)
);
CREATE TABLE IF NOT EXISTS event (
id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY,
created_time bigint NOT NULL,
body varchar(10000000),
entity_id uuid,
entity_type varchar(255),
event_type varchar(255),
event_uid varchar(255),
tenant_id uuid,
CREATE TABLE IF NOT EXISTS rule_node_debug_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL ,
ts bigint NOT NULL,
CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
);
entity_id uuid NOT NULL,
service_id varchar,
e_type varchar,
e_entity_id uuid,
e_entity_type varchar,
e_msg_id uuid,
e_msg_type varchar,
e_data_type varchar,
e_relation_type varchar,
e_data varchar,
e_metadata varchar,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS rule_chain_debug_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_message varchar,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS stats_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_messages_processed bigint NOT NULL,
e_errors_occurred bigint NOT NULL
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS lc_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_type varchar NOT NULL,
e_success boolean NOT NULL,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS error_event (
id uuid NOT NULL,
tenant_id uuid NOT NULL,
ts bigint NOT NULL,
entity_id uuid NOT NULL,
service_id varchar NOT NULL,
e_method varchar NOT NULL,
e_error varchar
) PARTITION BY RANGE (ts);
CREATE TABLE IF NOT EXISTS relation (
from_id uuid,

20
dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java

@ -28,17 +28,20 @@ import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.context.support.AnnotationConfigContextLoader;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
@ -185,17 +188,16 @@ public abstract class AbstractServiceTest {
}
protected Event generateEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) throws IOException {
protected RuleNodeDebugEvent generateEvent(TenantId tenantId, EntityId entityId) throws IOException {
if (tenantId == null) {
tenantId = TenantId.fromUUID(Uuids.timeBased());
}
Event event = new Event();
event.setTenantId(tenantId);
event.setEntityId(entityId);
event.setType(eventType);
event.setUid(eventUid);
event.setBody(readFromResource("TestJsonData.json"));
return event;
return RuleNodeDebugEvent.builder()
.tenantId(tenantId)
.entityId(entityId.getId())
.serviceId("server A")
.data(JacksonUtil.toString(readFromResource("TestJsonData.json")))
.build();
}
//
// private ComponentDescriptor getOrCreateDescriptor(ComponentScope scope, ComponentType type, String clazz, String configurationDescriptorResource) throws IOException {

68
dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java

@ -16,12 +16,13 @@
package org.thingsboard.server.dao.service.event;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.event.RuleNodeDebugEvent;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
@ -33,10 +34,7 @@ import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.service.AbstractServiceTest;
import java.text.ParseException;
import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneOffset;
import java.util.Optional;
import java.util.List;
import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT;
@ -58,15 +56,14 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest {
@Test
public void saveEvent() throws Exception {
TenantId tenantId = new TenantId(Uuids.timeBased());
DeviceId devId = new DeviceId(Uuids.timeBased());
Event event = generateEvent(null, devId, "ALARM", Uuids.timeBased().toString());
RuleNodeDebugEvent event = generateEvent(tenantId, devId);
eventService.saveAsync(event).get();
Optional<Event> loaded = eventService.findEvent(event.getTenantId(), event.getEntityId(), event.getType(), event.getUid());
Assert.assertTrue(loaded.isPresent());
Assert.assertNotNull(loaded.get());
Assert.assertEquals(event.getEntityId(), loaded.get().getEntityId());
Assert.assertEquals(event.getType(), loaded.get().getType());
Assert.assertEquals(event.getBody(), loaded.get().getBody());
List<EventInfo> loaded = eventService.findLatestEvents(event.getTenantId(), devId, event.getType(), 1);
Assert.assertNotNull(loaded);
Assert.assertEquals(1, loaded.size());
Assert.assertEquals(event.getData(), loaded.get(0).getBody().get("data").asText());
}
@Test
@ -79,25 +76,24 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest {
Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId);
saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId);
TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime);
TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts"), startTime, endTime);
PageData<Event> events = eventService.findEvents(tenantId, customerId, DataConstants.STATS,
timePageLink);
PageData<EventInfo> events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink);
Assert.assertNotNull(events.getData());
Assert.assertTrue(events.getData().size() == 2);
Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId()));
Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId()));
Assert.assertEquals(2, events.getData().size());
Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId());
Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId());
Assert.assertTrue(events.hasNext());
events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink());
events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink());
Assert.assertNotNull(events.getData());
Assert.assertTrue(events.getData().size() == 1);
Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId()));
Assert.assertEquals(1, events.getData().size());
Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId());
Assert.assertFalse(events.hasNext());
eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1);
eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true);
}
@Test
@ -110,30 +106,30 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest {
Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId);
saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId);
TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime);
TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts", SortOrder.Direction.DESC), startTime, endTime);
PageData<Event> events = eventService.findEvents(tenantId, customerId, DataConstants.STATS,
timePageLink);
PageData<EventInfo> events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink);
Assert.assertNotNull(events.getData());
Assert.assertTrue(events.getData().size() == 2);
Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId()));
Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId()));
Assert.assertEquals(2, events.getData().size());
Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId());
Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId());
Assert.assertTrue(events.hasNext());
events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink());
events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink());
Assert.assertNotNull(events.getData());
Assert.assertTrue(events.getData().size() == 1);
Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId()));
Assert.assertEquals(1, events.getData().size());
Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId());
Assert.assertFalse(events.hasNext());
eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1);
eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true);
}
private Event saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception {
Event event = generateEvent(tenantId, entityId, DataConstants.STATS, null);
event.setId(new EventId(Uuids.startOf(time)));
RuleNodeDebugEvent event = generateEvent(tenantId, entityId);
event.setId(new EventId(Uuids.timeBased()));
event.setCreatedTime(time);
eventService.saveAsync(event).get();
return event;
}

167
dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java

@ -16,39 +16,27 @@
package org.thingsboard.server.dao.sql.event;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.junit.After;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EventId;
import org.thingsboard.server.common.data.event.Event;
import org.thingsboard.server.common.data.event.EventType;
import org.thingsboard.server.common.data.event.StatisticsEvent;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.AbstractJpaDaoTest;
import org.thingsboard.server.dao.event.EventDao;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.thingsboard.server.common.data.DataConstants.ALARM;
import static org.thingsboard.server.common.data.DataConstants.STATS;
/**
* Created by Valerii Sosliuk on 5/5/2017.
*/
@Slf4j
public class JpaBaseEventDaoTest extends AbstractJpaDaoTest {
@ -56,21 +44,21 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest {
private EventDao eventDao;
UUID tenantId = Uuids.timeBased();
@After
public void deleteEvents() {
List<Event> events = eventDao.find(TenantId.fromUUID(tenantId));
for (Event event : events) {
eventDao.removeById(TenantId.fromUUID(tenantId), event.getUuidId());
}
}
@Test
public void findEvent() {
public void findEvent() throws InterruptedException, ExecutionException, TimeoutException {
UUID entityId = Uuids.timeBased();
Event savedEvent = eventDao.save(TenantId.fromUUID(tenantId), getEvent(entityId, tenantId, entityId));
Event foundEvent = eventDao.findEvent(tenantId, new DeviceId(entityId), DataConstants.STATS, savedEvent.getUid());
assertNotNull("Event expected to be not null", foundEvent);
assertEquals(savedEvent.getId(), foundEvent.getId());
Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId);
eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES);
Thread.sleep(2);
Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId);
eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES);
List<? extends Event> foundEvents = eventDao.findLatestEvents(tenantId, entityId, EventType.STATS, 1);
assertNotNull("Events expected to be not null", foundEvents);
assertEquals(1, foundEvents.size());
assertEquals(event2, foundEvents.get(0));
}
@Test
@ -78,108 +66,55 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest {
UUID entityId1 = Uuids.timeBased();
UUID entityId2 = Uuids.timeBased();
long startTime = System.currentTimeMillis();
long endTime = createEventsTwoEntities(tenantId, entityId1, entityId2, 20);
TimePageLink pageLink1 = new TimePageLink(30);
PageData<Event> events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink1);
assertEquals(10, events1.getData().size());
Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId1);
eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES);
Thread.sleep(2);
Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId2);
eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES);
TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null);
PageData<Event> events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink2);
assertEquals(10, events2.getData().size());
long endTime = System.currentTimeMillis();
TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime);
PageData<Event> events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink3);
assertEquals(10, events3.getData().size());
TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime);
PageData<Event> events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4);
assertEquals(5, events4.getData().size());
pageLink4 = pageLink4.nextPageLink();
PageData<Event> events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4);
assertEquals(5, events5.getData().size());
pageLink4 = pageLink4.nextPageLink();
PageData<Event> events6 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4);
assertEquals(0, events6.getData().size());
PageData<? extends Event> events1 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, new TimePageLink(30));
assertEquals(1, events1.getData().size());
}
PageData<? extends Event> events2 = eventDao.findEvents(tenantId, entityId2, EventType.STATS, new TimePageLink(30));
assertEquals(1, events2.getData().size());
@Test
public void findEventsByEntityIdAndEventTypeAndPageLink() throws Exception {
UUID entityId1 = Uuids.timeBased();
UUID entityId2 = Uuids.timeBased();
long startTime = System.currentTimeMillis();
long endTime = createEventsTwoEntitiesTwoTypes(tenantId, entityId1, entityId2, 20);
PageData<? extends Event> events3 = eventDao.findEvents(tenantId, Uuids.timeBased(), EventType.STATS, new TimePageLink(30));
assertEquals(0, events3.getData().size());
TimePageLink pageLink1 = new TimePageLink(30);
PageData<Event> events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink1);
assertEquals(5, events1.getData().size());
TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null);
PageData<Event> events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink2);
assertEquals(5, events2.getData().size());
PageData<? extends Event> events12 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink2);
assertEquals(1, events12.getData().size());
assertEquals(event1, events12.getData().get(0));
TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime);
PageData<Event> events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink3);
assertEquals(5, events3.getData().size());
PageData<? extends Event> events13 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink3);
assertEquals(1, events13.getData().size());
assertEquals(event1, events13.getData().get(0));
TimePageLink pageLink4 = new TimePageLink(4, 0, "", null, startTime, endTime);
PageData<Event> events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4);
assertEquals(4, events4.getData().size());
TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime);
PageData<? extends Event> events14 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4);
assertEquals(1, events14.getData().size());
assertEquals(event1, events14.getData().get(0));
pageLink4 = pageLink4.nextPageLink();
PageData<Event> events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4);
assertEquals(1, events5.getData().size());
}
private long createEventsTwoEntitiesTwoTypes(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception {
for (int i = 0; i < count / 2; i++) {
String type = i % 2 == 0 ? STATS : ALARM;
UUID eventId1 = Uuids.timeBased();
Event event1 = getEvent(eventId1, tenantId, entityId1, type);
eventDao.saveAsync(event1).get();
UUID eventId2 = Uuids.timeBased();
Event event2 = getEvent(eventId2, tenantId, entityId2, type);
eventDao.saveAsync(event2).get();
}
return System.currentTimeMillis();
}
private long createEventsTwoEntities(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception {
for (int i = 0; i < count / 2; i++) {
UUID eventId1 = Uuids.timeBased();
Event event1 = getEvent(eventId1, tenantId, entityId1);
eventDao.saveAsync(event1).get();
UUID eventId2 = Uuids.timeBased();
Event event2 = getEvent(eventId2, tenantId, entityId2);
eventDao.saveAsync(event2).get();
}
return System.currentTimeMillis();
}
PageData<? extends Event> events6 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4);
assertEquals(0, events6.getData().size());
private Event getEvent(UUID eventId, UUID tenantId, UUID entityId, String type) {
Event event = getEvent(eventId, tenantId, entityId);
event.setType(type);
return event;
}
private Event getEvent(UUID eventId, UUID tenantId, UUID entityId) {
Event event = new Event();
event.setId(new EventId(eventId));
event.setTenantId(TenantId.fromUUID(tenantId));
EntityId deviceId = new DeviceId(entityId);
event.setEntityId(deviceId);
event.setUid(event.getId().getId().toString());
event.setType(STATS);
ObjectMapper mapper = new ObjectMapper();
try {
JsonNode jsonNode = mapper.readTree("{\"key\":\"value\"}");
event.setBody(jsonNode);
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return event;
private Event getStatsEvent(UUID eventId, UUID tenantId, UUID entityId) {
StatisticsEvent.StatisticsEventBuilder event = StatisticsEvent.builder();
event.id(eventId);
event.ts(System.currentTimeMillis());
event.tenantId(new TenantId(tenantId));
event.entityId(entityId);
event.serviceId("server A");
event.messagesProcessed(1);
event.errorsOccurred(0);
return event.build();
}
}

2
dao/src/test/resources/cassandra-test.properties

@ -58,8 +58,6 @@ cassandra.query.ts_key_value_partitions_max_cache_size=100000
cassandra.query.ts_key_value_ttl=0
cassandra.query.debug_events_ttl=604800
cassandra.query.max_limit_per_request=1000
cassandra.query.buffer_size=100000
cassandra.query.concurrent_limit=1000

10
rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java

@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.EntityViewInfo;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.EventInfo;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest;
@ -1811,7 +1811,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
}
}
public PageData<Event> getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) {
public PageData<EventInfo> getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) {
Map<String, String> params = new HashMap<>();
params.put("entityType", entityId.getEntityType().name());
params.put("entityId", entityId.getId().toString());
@ -1823,12 +1823,12 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
baseURL + "/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&" + getTimeUrlParams(pageLink),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<Event>>() {
new ParameterizedTypeReference<PageData<EventInfo>>() {
},
params).getBody();
}
public PageData<Event> getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) {
public PageData<EventInfo> getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) {
Map<String, String> params = new HashMap<>();
params.put("entityType", entityId.getEntityType().name());
params.put("entityId", entityId.getId().toString());
@ -1839,7 +1839,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable {
baseURL + "/api/events/{entityType}/{entityId}?tenantId={tenantId}&" + getTimeUrlParams(pageLink),
HttpMethod.GET,
HttpEntity.EMPTY,
new ParameterizedTypeReference<PageData<Event>>() {
new ParameterizedTypeReference<PageData<EventInfo>>() {
}, params).getBody();
}

2
ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts

@ -75,7 +75,7 @@ export class EventFilterPanelComponent {
}
isNumberFields(key: string): string {
return ['messagesProcessed', 'errorsOccurred'].includes(key) ? key : '';
return ['minMessagesProcessed', 'maxMessagesProcessed', 'minErrorsOccurred', 'maxErrorsOccurred'].includes(key) ? key : '';
}
selectorValues(key: string): string[] {

45
ui-ngx/src/app/modules/home/components/event/event-table-config.ts

@ -225,7 +225,6 @@ export class EventTableConfig extends EntityTableConfig<Event, TimePageLink> {
);
break;
case DebugEventType.DEBUG_RULE_NODE:
case DebugEventType.DEBUG_RULE_CHAIN:
this.columns[0].width = '100px';
this.columns.push(
new EntityTableColumn<Event>('type', 'event.type', '40px',
@ -234,8 +233,8 @@ export class EventTableConfig extends EntityTableConfig<Event, TimePageLink> {
}), false, key => ({
padding: '0 12px 0 0'
})),
new EntityTableColumn<Event>('entityName', 'event.entity-type', '100px',
(entity) => entity.body.entityName, entity => ({
new EntityTableColumn<Event>('entityType', 'event.entity-type', '100px',
(entity) => entity.body.entityType, entity => ({
padding: '0 12px 0 0',
}), false, key => ({
padding: '0 12px 0 0'
@ -289,6 +288,29 @@ export class EventTableConfig extends EntityTableConfig<Event, TimePageLink> {
'40px')
);
break;
case DebugEventType.DEBUG_RULE_CHAIN:
this.columns[0].width = '100px';
this.columns.push(
new EntityActionTableColumn<Event>('message', 'event.message',
{
name: this.translate.instant('action.view'),
icon: 'more_horiz',
isEnabled: (entity) => entity.body.message ? entity.body.message.length > 0 : false,
onAction: ($event, entity) => this.showContent($event, entity.body.message,
'event.message')
},
'40px'),
new EntityActionTableColumn<Event>('error', 'event.error',
{
name: this.translate.instant('action.view'),
icon: 'more_horiz',
isEnabled: (entity) => entity.body.error && entity.body.error.length > 0,
onAction: ($event, entity) => this.showContent($event, entity.body.error,
'event.error')
},
'40px')
);
break;
}
if (updateTableColumns) {
this.getTable().columnsUpdated(true);
@ -334,16 +356,18 @@ export class EventTableConfig extends EntityTableConfig<Event, TimePageLink> {
break;
case EventType.STATS:
this.filterColumns.push(
{key: 'messagesProcessed', title: 'event.min-messages-processed'},
{key: 'errorsOccurred', title: 'event.min-errors-occurred'}
{key: 'minMessagesProcessed', title: 'event.min-messages-processed'},
{key: 'maxMessagesProcessed', title: 'event.max-messages-processed'},
{key: 'minErrorsOccurred', title: 'event.min-errors-occurred'},
{key: 'maxErrorsOccurred', title: 'event.max-errors-occurred'}
);
break;
case DebugEventType.DEBUG_RULE_NODE:
case DebugEventType.DEBUG_RULE_CHAIN:
this.filterColumns.push(
{key: 'msgDirectionType', title: 'event.type'},
{key: 'entityId', title: 'event.entity-id'},
{key: 'entityName', title: 'event.entity-type'},
{key: 'entityType', title: 'event.entity-type'},
{key: 'msgId', title: 'event.message-id'},
{key: 'msgType', title: 'event.message-type'},
{key: 'relationType', title: 'event.relation-type'},
{key: 'dataSearch', title: 'event.data'},
@ -352,6 +376,13 @@ export class EventTableConfig extends EntityTableConfig<Event, TimePageLink> {
{key: 'errorStr', title: 'event.error'}
);
break;
case DebugEventType.DEBUG_RULE_CHAIN:
this.filterColumns.push(
{key: 'message', title: 'event.message'},
{key: 'isError', title: 'event.error'},
{key: 'errorStr', title: 'event.error'}
);
break;
}
}

9
ui-ngx/src/app/shared/models/event.models.ts

@ -65,7 +65,7 @@ export interface StatsEventBody extends BaseEventBody {
export interface DebugRuleNodeEventBody extends BaseEventBody {
type: string;
entityId: string;
entityName: string;
entityType: string;
msgId: string;
msgType: string;
relationType: string;
@ -75,7 +75,12 @@ export interface DebugRuleNodeEventBody extends BaseEventBody {
error: string;
}
export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody;
export interface DebugRuleChainEventBody extends BaseEventBody {
message: string;
error?: string;
}
export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody;
export interface Event extends BaseData<EventId> {
tenantId: TenantId;

3
ui-ngx/src/assets/locale/locale.constant-en_US.json

@ -2012,6 +2012,7 @@
"body": "Body",
"method": "Method",
"type": "Type",
"message": "Message",
"message-id": "Message Id",
"message-type": "Message Type",
"data-type": "Data Type",
@ -2023,8 +2024,10 @@
"success": "Success",
"failed": "Failed",
"messages-processed": "Messages processed",
"max-messages-processed": "Maximum messages processed",
"min-messages-processed": "Minimum messages processed",
"errors-occurred": "Errors occurred",
"max-errors-occurred": "Maximum errors occurred",
"min-errors-occurred": "Minimum errors occurred",
"min-value": "Minimum value is 0.",
"all-events": "All",

Loading…
Cancel
Save