diff --git a/application/src/main/data/upgrade/3.4.0/schema_update.sql b/application/src/main/data/upgrade/3.4.0/schema_update.sql new file mode 100644 index 0000000000..12a86acebc --- /dev/null +++ b/application/src/main/data/upgrade/3.4.0/schema_update.sql @@ -0,0 +1,230 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS rule_node_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar, + e_type varchar, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_data_type varchar, + e_relation_type varchar, + e_data varchar, + e_metadata varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_message varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS stats_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_messages_processed bigint NOT NULL, + e_errors_occurred bigint NOT NULL +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS lc_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_type varchar NOT NULL, + e_success boolean NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS error_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_method varchar NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_stats_event_main + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_lc_event_main + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_error_event_main + ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE OR REPLACE FUNCTION to_safe_json(p_json text) RETURNS json +LANGUAGE plpgsql AS +$$ +BEGIN + return REPLACE(p_json, '\u0000', '' )::json; +EXCEPTION + WHEN OTHERS THEN + return '{}'::json; +END; +$$; + +-- Useful to migrate old events to the new table structure; +CREATE OR REPLACE PROCEDURE migrate_regular_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int) + LANGUAGE plpgsql AS +$$ +DECLARE + partition_size_in_ms bigint; + p record; + table_name varchar; +BEGIN + partition_size_in_ms = partition_size_in_hours * 3600 * 1000; + + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('STATS', 'LC_EVENT', 'ERROR') and ts >= start_ts_in_ms and ts < end_ts_in_ms + LOOP + IF p.event_type = 'STATS' THEN + table_name := 'stats_event'; + ELSEIF p.event_type = 'LC_EVENT' THEN + table_name := 'lc_event'; + ELSEIF p.event_type = 'ERROR' THEN + table_name := 'error_event'; + END IF; + RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms); + EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms)); + END LOOP; + + INSERT INTO stats_event + SELECT id, + tenant_id, + ts, + entity_id, + body ->> 'server', + (body ->> 'messagesProcessed')::bigint, + (body ->> 'errorsOccurred')::bigint + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'STATS' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event + ON CONFLICT DO NOTHING; + + INSERT INTO lc_event + SELECT id, + tenant_id, + ts, + entity_id, + body ->> 'server', + body ->> 'event', + (body ->> 'success')::boolean, + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'LC_EVENT' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event + ON CONFLICT DO NOTHING; + + INSERT INTO error_event + SELECT id, + tenant_id, + ts, + entity_id, + body ->> 'server', + body ->> 'method', + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'ERROR' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event + ON CONFLICT DO NOTHING; + +END +$$; + +-- Useful to migrate old debug events to the new table structure; +CREATE OR REPLACE PROCEDURE migrate_debug_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int) + LANGUAGE plpgsql AS +$$ +DECLARE + partition_size_in_ms bigint; + p record; + table_name varchar; +BEGIN + partition_size_in_ms = partition_size_in_hours * 3600 * 1000; + + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') and ts >= start_ts_in_ms and ts < end_ts_in_ms + LOOP + IF p.event_type = 'DEBUG_RULE_NODE' THEN + table_name := 'rule_node_debug_event'; + ELSEIF p.event_type = 'DEBUG_RULE_CHAIN' THEN + table_name := 'rule_chain_debug_event'; + END IF; + RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms); + EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms)); + END LOOP; + + INSERT INTO rule_node_debug_event + SELECT id, + tenant_id, + ts, + entity_id, + body ->> 'server', + body ->> 'type', + (body ->> 'entityId')::uuid, + body ->> 'entityName', + (body ->> 'msgId')::uuid, + body ->> 'msgType', + body ->> 'dataType', + body ->> 'relationType', + body ->> 'data', + body ->> 'metadata', + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_NODE' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event + ON CONFLICT DO NOTHING; + + INSERT INTO rule_chain_debug_event + SELECT id, + tenant_id, + ts, + entity_id, + body ->> 'server', + body ->> 'message', + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_CHAIN' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event + ON CONFLICT DO NOTHING; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index a38de7b839..b4c308db86 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -37,10 +37,11 @@ import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; @@ -112,6 +113,29 @@ import java.util.concurrent.TimeUnit; @Component public class ActorSystemContext { + private static final FutureCallback RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Rule Chain", th); + } + }; + private static final FutureCallback RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Node", th); + } + }; + protected final ObjectMapper mapper = new ObjectMapper(); private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); @@ -462,25 +486,28 @@ public class ActorSystemContext { } public void persistError(TenantId tenantId, EntityId entityId, String method, Exception e) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); - persistEvent(event); + eventService.saveAsync(ErrorEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .method(method) + .error(toString(e)).build()); } public void persistLifecycleEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent lcEvent, Exception e) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); - persistEvent(event); - } + LifecycleEvent.LifecycleEventBuilder event = LifecycleEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .lcEventType(lcEvent.name()); + + if (e != null) { + event.success(false).error(toString(e)); + } else { + event.success(true); + } - private void persistEvent(Event event) { - eventService.saveAsync(event); + eventService.saveAsync(event.build()); } private String toString(Throwable e) { @@ -489,21 +516,6 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); - if (e.isPresent()) { - node = node.put("success", false); - node = node.put("error", toString(e.get())); - } else { - node = node.put("success", true); - } - return node; - } - - private JsonNode toBodyJson(String serviceId, String method, String body) { - return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); - } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return partitionService.resolve(serviceType, tenantId, entityId); } @@ -539,44 +551,27 @@ public class ActorSystemContext { private void persistDebugAsync(TenantId tenantId, EntityId entityId, String type, TbMsg tbMsg, String relationType, Throwable error, String failureMessage) { if (checkLimits(tenantId, tbMsg, error)) { try { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.DEBUG_RULE_NODE); - - String metadata = mapper.writeValueAsString(tbMsg.getMetaData().getData()); - - ObjectNode node = mapper.createObjectNode() - .put("type", type) - .put("server", getServiceId()) - .put("entityId", tbMsg.getOriginator().getId().toString()) - .put("entityName", tbMsg.getOriginator().getEntityType().name()) - .put("msgId", tbMsg.getId().toString()) - .put("msgType", tbMsg.getType()) - .put("dataType", tbMsg.getDataType().name()) - .put("relationType", relationType) - .put("data", tbMsg.getData()) - .put("metadata", metadata); + RuleNodeDebugEvent.RuleNodeDebugEventBuilder event = RuleNodeDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .eventType(type) + .eventEntity(tbMsg.getOriginator()) + .msgId(tbMsg.getId()) + .msgType(tbMsg.getType()) + .dataType(tbMsg.getDataType().name()) + .relationType(relationType) + .data(tbMsg.getData()) + .metadata(mapper.writeValueAsString(tbMsg.getMetaData().getData())); if (error != null) { - node = node.put("error", toString(error)); + event.error(toString(error)); } else if (failureMessage != null) { - node = node.put("error", failureMessage); + event.error(failureMessage); } - event.setBody(node); - ListenableFuture future = eventService.saveAsync(event); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void event) { - - } - - @Override - public void onFailure(Throwable th) { - log.error("Could not save debug Event for Node", th); - } - }, MoreExecutors.directExecutor()); + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } catch (IOException ex) { log.warn("Failed to persist rule node debug message", ex); } @@ -603,33 +598,17 @@ public class ActorSystemContext { } private void persistRuleChainDebugModeEvent(TenantId tenantId, EntityId entityId, Throwable error) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.DEBUG_RULE_CHAIN); - - ObjectNode node = mapper.createObjectNode() - //todo: what fields are needed here? - .put("server", getServiceId()) - .put("message", "Reached debug mode rate limit!"); - + RuleChainDebugEvent.RuleChainDebugEventBuilder event = RuleChainDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId(getServiceId()) + .message("Reached debug mode rate limit!"); if (error != null) { - node = node.put("error", toString(error)); + event.error(toString(error)); } - event.setBody(node); - ListenableFuture future = eventService.saveAsync(event); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void event) { - - } - - @Override - public void onFailure(Throwable th) { - log.error("Could not save debug Event for Rule Chain", th); - } - }, MoreExecutors.directExecutor()); + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } public static Exception toException(Throwable error) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 8cdf5cec02..247a6c62d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -20,13 +20,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActor; -import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorId; import org.thingsboard.server.actors.TbStringActorId; import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; @@ -54,12 +54,14 @@ public class StatsActor extends ContextAwareActor { if (msg.isEmpty()) { return; } - Event event = new Event(); - event.setEntityId(msg.getEntityId()); - event.setTenantId(msg.getTenantId()); - event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); - systemContext.getEventService().saveAsync(event); + systemContext.getEventService().saveAsync(StatisticsEvent.builder() + .tenantId(msg.getTenantId()) + .entityId(msg.getEntityId().getId()) + .serviceId(systemContext.getServiceInfoProvider().getServiceId()) + .messagesProcessed(msg.getMessagesProcessed()) + .errorsOccurred(msg.getErrorsOccurred()) + .build() + ); } private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index bdf74c9055..d4199006d0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -94,7 +94,7 @@ public class ControllerConstants { protected static final String DEVICE_PROFILE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, transportType, description, isDefault"; protected static final String ASSET_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle"; protected static final String ALARM_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, startTs, endTs, type, ackTs, clearTs, severity, status"; - protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, id"; + protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "ts, id"; protected static final String EDGE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle"; protected static final String RULE_CHAIN_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, root"; protected static final String WIDGET_BUNDLE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, title, tenantId"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index ac56978536..45042dc883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -29,8 +29,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -42,6 +45,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Locale; + import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE; @@ -110,7 +115,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}/{eventType}", method = RequestMethod.GET) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @@ -135,16 +140,12 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEvents(tenantId, entityId, eventType, pageLink)); - } catch (Exception e) { - throw handleException(e); - } + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + return checkNotNull(eventService.findEvents(tenantId, entityId, resolveEventType(eventType), pageLink)); } @ApiOperation(value = "Get Events (getEvents)", @@ -153,7 +154,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.GET) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @@ -176,18 +177,14 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEvents(tenantId, entityId, pageLink)); - } catch (Exception e) { - throw handleException(e); - } + return checkNotNull(eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, pageLink)); } @ApiOperation(value = "Get Events by event filter (getEvents)", @@ -198,7 +195,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.POST) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @@ -223,21 +220,17 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - if (sortProperty != null && sortProperty.equals("createdTime") && eventFilter.hasFilterForJsonBody()) { - sortProperty = ModelConstants.CREATED_TIME_PROPERTY; - } + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink)); - } catch (Exception e) { - throw handleException(e); + if (sortProperty != null && sortProperty.equals("createdTime")) { + sortProperty = ModelConstants.TS_COLUMN; } + + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink)); } @ApiOperation(value = "Clear Events (clearEvents)", notes = "Clears events by filter for specified entity.") @@ -266,4 +259,13 @@ public class EventController extends BaseController { } } + private static EventType resolveEventType(String eventType) throws ThingsboardException { + for (var et : EventType.values()) { + if (et.name().equalsIgnoreCase(eventType) || et.getOldName().equalsIgnoreCase(eventType)) { + return et; + } + } + throw new ThingsboardException("Event type: '" + eventType + "' is not supported!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index faa1711fd9..5c79f9eca7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -39,10 +39,10 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -70,7 +70,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -353,10 +352,10 @@ public class RuleChainController extends BaseController { RuleNodeId ruleNodeId = new RuleNodeId(toUUID(strRuleNodeId)); checkRuleNode(ruleNodeId, Operation.READ); TenantId tenantId = getCurrentUser().getTenantId(); - List events = eventService.findLatestEvents(tenantId, ruleNodeId, DataConstants.DEBUG_RULE_NODE, 2); + List events = eventService.findLatestEvents(tenantId, ruleNodeId, EventType.DEBUG_RULE_NODE, 2); JsonNode result = null; if (events != null) { - for (Event event : events) { + for (EventInfo event : events) { JsonNode body = event.getBody(); if (body.has("type") && body.get("type").asText().equals("IN")) { result = body; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 40e15c7b0e..2c3251834f 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -221,6 +221,10 @@ public class ThingsboardInstallService { log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); dataUpdateService.updateData("3.3.4"); + case "3.4.0": + log.info("Upgrading ThingsBoard from version 3.4.0 to 3.4.1 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.4.0"); + dataUpdateService.updateData("3.4.0"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index d22039683d..ea976b2dec 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -25,6 +25,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.QueueId; @@ -596,6 +597,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.4.0": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.0", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004001;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } @@ -666,5 +679,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } - } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b90da586fc..1eda4e027d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -58,6 +58,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.relation.RelationService; @@ -128,6 +129,9 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private SystemDataLoaderService systemDataLoaderService; + @Autowired + private EventService eventService; + @Override public void updateData(String fromVersion) throws Exception { switch (fromVersion) { @@ -159,6 +163,13 @@ public class DefaultDataUpdateService implements DataUpdateService { tenantsProfileQueueConfigurationUpdater.updateEntities(); rateLimitsUpdater.updateEntities(); break; + case "3.4.0": + String skipEventsMigration = System.getenv("TB_SKIP_EVENTS_MIGRATION"); + if (skipEventsMigration == null || skipEventsMigration.equalsIgnoreCase("false")) { + log.info("Updating data from version 3.4.0 to 3.4.1 ..."); + eventService.migrateEvents(); + } + break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } @@ -602,7 +613,7 @@ public class DefaultDataUpdateService implements DataUpdateService { }); } } catch (Exception e) { - log.error("Failed to update tenant profile queue configuration name=["+profile.getName()+"], id=["+ profile.getId().getId() +"]", e); + log.error("Failed to update tenant profile queue configuration name=[" + profile.getName() + "], id=[" + profile.getId().getId() + "]", e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index 1fd3892f39..3dc66b6210 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -25,7 +25,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.concurrent.TimeUnit; -@TbCoreComponent @Slf4j @Service public class EventsCleanUpService extends AbstractCleanUpService { @@ -39,9 +38,6 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Value("${sql.ttl.events.debug_events_ttl}") private long debugTtlInSec; - @Value("${sql.ttl.events.execution_interval_ms}") - private long executionIntervalInMs; - @Value("${sql.ttl.events.enabled}") private boolean ttlTaskExecutionEnabled; @@ -54,28 +50,11 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.events.execution_interval_ms}") public void cleanUp() { - if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { + if (ttlTaskExecutionEnabled) { long ts = System.currentTimeMillis(); - long regularEventStartTs; - long regularEventEndTs; - long debugEventStartTs; - long debugEventEndTs; - - if (ttlInSec > 0) { - regularEventEndTs = ts - TimeUnit.SECONDS.toMillis(ttlInSec); - regularEventStartTs = regularEventEndTs - 2 * executionIntervalInMs; - } else { - regularEventStartTs = regularEventEndTs = 0; - } - - if (debugTtlInSec > 0) { - debugEventEndTs = ts - TimeUnit.SECONDS.toMillis(debugTtlInSec); - debugEventStartTs = debugEventEndTs - 2 * executionIntervalInMs; - } else { - debugEventStartTs = debugEventEndTs = 0; - } - - eventService.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + long regularEventExpTs = ttlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(ttlInSec) : 0; + long debugEventExpTs = debugTtlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(debugTtlInSec) : 0; + eventService.cleanupEvents(regularEventExpTs, debugEventExpTs, isSystemTenantPartitionMine()); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 17b996934e..1c50fb9ced 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -221,9 +221,6 @@ cassandra: ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}" ts_key_value_ttl: "${TS_KV_TTL:0}" - events_ttl: "${TS_EVENTS_TTL:0}" - # Specify TTL of debug log in seconds. The current value corresponds to one week - debug_events_ttl: "${DEBUG_EVENTS_TTL:604800}" buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}" concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}" permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}" @@ -262,6 +259,8 @@ sql: batch_max_delay: "${SQL_EVENTS_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_EVENTS_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_EVENTS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution + partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week. + debug_partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:1}" # Number of hours to partition the debug events. The current value corresponds to one hour. edge_events: batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" @@ -287,9 +286,11 @@ sql: ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:3600000}" # Number of milliseconds (max random initial delay and fixed period). + # Number of seconds. TTL is disabled by default. Accuracy of the cleanup depends on the sql.events.partition_size parameter. + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" + # Number of seconds. The current value corresponds to one week. Accuracy of the cleanup depends on the sql.events.debug_partition_size parameter. + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" edge_events: enabled: "${SQL_TTL_EDGE_EVENTS_ENABLED:true}" execution_interval_ms: "${SQL_TTL_EDGE_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day diff --git a/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java b/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java index 0e928b7dd6..d4d9456a3f 100644 --- a/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java @@ -18,7 +18,8 @@ package org.thingsboard.server.actors.stats; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index ab902b82c6..a3edae8ed6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -19,8 +19,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -60,20 +60,19 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle return doGet("/api/ruleChain/metadata/" + ruleChainId.getId().toString(), RuleChainMetaData.class); } - protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { - return getEvents(tenantId, entityId, DataConstants.DEBUG_RULE_NODE, limit); + protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { + return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE.getOldName(), limit); } - protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { + protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { TimePageLink pageLink = new TimePageLink(limit); return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&", - new TypeReference>() { + new TypeReference>() { }, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId()); } - - protected JsonNode getMetadata(Event outEvent) { + protected JsonNode getMetadata(EventInfo outEvent) { String metaDataStr = outEvent.getBody().get("metadata").asText(); try { return mapper.readTree(metaDataStr); @@ -82,7 +81,7 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle } } - protected Predicate filterByCustomEvent() { + protected Predicate filterByCustomEvent() { return event -> event.getBody().get("msgType").textValue().equals("CUSTOM"); } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 9d96fe9d9c..24c94adba6 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -30,9 +30,10 @@ import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; @@ -177,15 +178,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule actorSystem.tell(qMsg); Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); - PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); - List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); + PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); + List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); Assert.assertEquals(2, events.size()); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(ruleChain.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(ruleChain.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); @@ -299,16 +300,16 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); - PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); - List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); + PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); + List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); Assert.assertEquals(2, events.size()); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 5d86e6a2a5..70cd292e00 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -20,18 +20,15 @@ import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.test.context.support.TestPropertySourceUtils; -import org.testcontainers.containers.GenericContainer; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.rule.RuleChain; @@ -109,11 +106,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Assert.assertNotNull(ruleChainFinal.getFirstRuleNodeId()); //TODO find out why RULE_NODE update event did not appear all the time - List rcEvents = Awaitility.await("Rule Node started successfully") + List rcEvents = Awaitility.await("Rule Node started successfully") .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000) + List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), EventType.LC_EVENT.getOldName(), 1000) .getData().stream().filter(e -> { var body = e.getBody(); return body.has("event") && body.get("event").asText().equals("STARTED") @@ -145,11 +142,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac log.warn("awaiting tbMsgCallback"); Mockito.verify(tbMsgCallback, Mockito.timeout(TimeUnit.SECONDS.toMillis(TIMEOUT))).onSuccess(); log.warn("awaiting events"); - List events = Awaitility.await("get debug by custom event") + List events = Awaitility.await("get debug by custom event") .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000) + List debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000) .getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); log.warn("filtered debug events [{}]", debugEvents.size()); debugEvents.forEach((e) -> log.warn("event: {}", e)); @@ -158,11 +155,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac x -> x.size() == 2); log.warn("asserting.."); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 83624df8e3..3b0de80811 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -16,34 +16,32 @@ package org.thingsboard.server.dao.event; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import java.util.List; -import java.util.Optional; public interface EventService { ListenableFuture saveAsync(Event event); - Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid); + PageData findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink); - PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink); + List findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit); - PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink); - - List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); - - PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); + PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); void removeEvents(TenantId tenantId, EntityId entityId); void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime); - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb); + void migrateEvents(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 1351621bf5..90d95fb561 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -52,12 +52,6 @@ public class DataConstants { } public static final String ALARM = "ALARM"; - public static final String ERROR = "ERROR"; - public static final String LC_EVENT = "LC_EVENT"; - public static final String STATS = "STATS"; - public static final String DEBUG_RULE_NODE = "DEBUG_RULE_NODE"; - public static final String DEBUG_RULE_CHAIN = "DEBUG_RULE_CHAIN"; - public static final String IN = "IN"; public static final String OUT = "OUT"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java similarity index 93% rename from common/data/src/main/java/org/thingsboard/server/common/data/Event.java rename to common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java index 29ab33d075..f6c4ee69fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Event.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; */ @Data @ApiModel -public class Event extends BaseData { +public class EventInfo extends BaseData { @ApiModelProperty(position = 1, value = "JSON object with Tenant Id.", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private TenantId tenantId; @@ -41,15 +41,15 @@ public class Event extends BaseData { @ApiModelProperty(position = 5, value = "Event body.", dataType = "com.fasterxml.jackson.databind.JsonNode") private transient JsonNode body; - public Event() { + public EventInfo() { super(); } - public Event(EventId id) { + public EventInfo(EventId id) { super(id); } - public Event(Event event) { + public EventInfo(EventInfo event) { super(event); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java new file mode 100644 index 0000000000..c895b61c78 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.springframework.data.domain.Page; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.StringUtils; + +import java.util.UUID; + +@Data +@ApiModel +public abstract class DebugEventFilter implements EventFilter { + + @ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") + protected String server; + @ApiModelProperty(position = 10, value = "Boolean value to filter the errors", allowableValues = "false, true") + protected boolean isError; + @ApiModelProperty(position = 11, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB") + protected String errorStr; + + public void setIsError(boolean isError) { + this.isError = isError; + } + + @Override + public boolean isNotEmpty() { + return !StringUtils.isEmpty(server) || isError || !StringUtils.isEmpty(errorStr); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java deleted file mode 100644 index ec55c8d1a2..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.event; - -import io.swagger.annotations.ApiModel; - -@ApiModel -public class DebugRuleNodeEventFilter extends DebugEvent { - @Override - public EventType getEventType() { - return EventType.DEBUG_RULE_NODE; - } -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java new file mode 100644 index 0000000000..341548df59 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java @@ -0,0 +1,66 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class ErrorEvent extends Event { + + private static final long serialVersionUID = 960461434033192571L; + + @Builder + private ErrorEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String method, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.method = method; + this.error = error; + } + + @Getter + @Setter + private String method; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.ERROR; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("method", method); + if (error != null) { + json.put("error", error); + } + return eventInfo; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java index 9332cd283a..c8f2106764 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java @@ -37,7 +37,7 @@ public class ErrorEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { + public boolean isNotEmpty() { return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(method) || !StringUtils.isEmpty(errorStr); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java new file mode 100644 index 0000000000..801289791a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +public abstract class Event extends BaseData { + + protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + protected final TenantId tenantId; + protected final UUID entityId; + protected final String serviceId; + + public Event(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts) { + super(); + if (id != null) { + this.id = new EventId(id); + } + this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID; + this.entityId = entityId; + this.serviceId = serviceId; + this.createdTime = ts; + } + + public abstract EventType getType(); + + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = new EventInfo(); + eventInfo.setTenantId(tenantId); + eventInfo.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + eventInfo.setType(getType().getOldName()); + eventInfo.setId(id); + eventInfo.setUid(id.toString()); + eventInfo.setCreatedTime(createdTime); + eventInfo.setBody(OBJECT_MAPPER.createObjectNode().put("server", getServiceId())); + return eventInfo; + } + + protected static void putNotNull(ObjectNode json, String key, String value) { + if (value != null) { + json.put(key, value); + } + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 5d02fbcd13..39502eba3b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -26,8 +26,8 @@ import io.swagger.annotations.ApiModelProperty; include = JsonTypeInfo.As.PROPERTY, property = "eventType") @JsonSubTypes({ - @JsonSubTypes.Type(value = DebugRuleNodeEventFilter.class, name = "DEBUG_RULE_NODE"), - @JsonSubTypes.Type(value = DebugRuleChainEventFilter.class, name = "DEBUG_RULE_CHAIN"), + @JsonSubTypes.Type(value = RuleNodeDebugEventFilter.class, name = "DEBUG_RULE_NODE"), + @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") @@ -37,6 +37,6 @@ public interface EventFilter { @ApiModelProperty(position = 1, required = true, value = "String value representing the event type", example = "STATS") EventType getEventType(); - boolean hasFilterForJsonBody(); + boolean isNotEmpty(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 2645e7cce3..a7909d28b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -15,6 +15,30 @@ */ package org.thingsboard.server.common.data.event; +import lombok.Getter; + public enum EventType { - ERROR, LC_EVENT, STATS, DEBUG_RULE_NODE, DEBUG_RULE_CHAIN + ERROR("error_event", "ERROR"), + LC_EVENT("lc_event", "LC_EVENT"), + STATS("stats_event", "STATS"), + DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); + + @Getter + private final String table; + @Getter + private final String oldName; + @Getter + private final boolean debug; + + EventType(String table, String oldName) { + this(table, oldName, false); + } + + EventType(String table, String oldName, boolean debug) { + this.table = table; + this.oldName = oldName; + this.debug = debug; + } + } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java index de50f1579f..104939d202 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java @@ -39,7 +39,7 @@ public class LifeCycleEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { + public boolean isNotEmpty() { return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(event) || !StringUtils.isEmpty(status) || !StringUtils.isEmpty(errorStr); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java new file mode 100644 index 0000000000..1e78df36a6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.jetbrains.annotations.NotNull; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class LifecycleEvent extends Event { + + private static final long serialVersionUID = -3247420461850911549L; + + @Builder + private LifecycleEvent(TenantId tenantId, UUID entityId, String serviceId, + UUID id, long ts, + String lcEventType, boolean success, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.lcEventType = lcEventType; + this.success = success; + this.error = error; + } + + @Getter + private final String lcEventType; + @Getter + private final boolean success; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.LC_EVENT; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("event", lcEventType) + .put("success", success); + if (error != null) { + json.put("error", error); + } + return eventInfo; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java new file mode 100644 index 0000000000..7de37d1b5b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class RuleChainDebugEvent extends Event { + + private static final long serialVersionUID = -386392236201116767L; + + @Builder + private RuleChainDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String message, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.message = message; + this.error = error; + } + + @Getter + @Setter + private String message; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_RULE_CHAIN; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + putNotNull(json, "message", message); + putNotNull(json, "error", error); + return eventInfo; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java similarity index 61% rename from common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java rename to common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java index 3569e097d0..f40471a35f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java @@ -16,11 +16,26 @@ package org.thingsboard.server.common.data.event; import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; +@Data +@EqualsAndHashCode(callSuper = true) @ApiModel -public class DebugRuleChainEventFilter extends DebugEvent { +public class RuleChainDebugEventFilter extends DebugEventFilter { + + @ApiModelProperty(position = 2, value = "String value representing the message") + protected String message; + @Override public EventType getEventType() { return EventType.DEBUG_RULE_CHAIN; } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(message); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java new file mode 100644 index 0000000000..ee87d976cf --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class RuleNodeDebugEvent extends Event { + + private static final long serialVersionUID = -6575797430064573984L; + + @Builder + private RuleNodeDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, + String eventType, EntityId eventEntity, UUID msgId, + String msgType, String dataType, String relationType, + String data, String metadata, String error) { + super(tenantId, entityId, serviceId, id, ts); + this.eventType = eventType; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.dataType = dataType; + this.relationType = relationType; + this.data = data; + this.metadata = metadata; + this.error = error; + } + + @Getter + private final String eventType; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + private final String dataType; + @Getter + private final String relationType; + @Getter + @Setter + private String data; + @Getter + @Setter + private String metadata; + @Getter + @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_RULE_NODE; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("type", eventType); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "dataType", dataType); + putNotNull(json, "relationType", relationType); + putNotNull(json, "data", data); + putNotNull(json, "metadata", metadata); + putNotNull(json, "error", error); + return eventInfo; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java similarity index 59% rename from common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java rename to common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java index 90bbff0bd5..aa44141365 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java @@ -18,41 +18,40 @@ package org.thingsboard.server.common.data.event; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.StringUtils; @Data +@EqualsAndHashCode(callSuper = true) @ApiModel -public abstract class DebugEvent implements EventFilter { +public class RuleNodeDebugEventFilter extends DebugEventFilter { - @ApiModelProperty(position = 1, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT") + @ApiModelProperty(position = 2, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT") protected String msgDirectionType; - @ApiModelProperty(position = 2, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") - protected String server; - @ApiModelProperty(position = 3, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity") - protected String dataSearch; - @ApiModelProperty(position = 4, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName") - protected String metadataSearch; - @ApiModelProperty(position = 5, value = "String value representing the entity type", allowableValues = "DEVICE") - protected String entityName; - @ApiModelProperty(position = 6, value = "String value representing the type of message routing", example = "Success") - protected String relationType; - @ApiModelProperty(position = 7, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") + @ApiModelProperty(position = 3, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") protected String entityId; - @ApiModelProperty(position = 8, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + @ApiModelProperty(position = 4, value = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @ApiModelProperty(position = 5, value = "String value representing the message id in the rule engine", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") + protected String msgId; + @ApiModelProperty(position = 6, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") protected String msgType; - @ApiModelProperty(position = 9, value = "Boolean value to filter the errors", allowableValues = "false, true") - protected boolean isError; - @ApiModelProperty(position = 10, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB") - protected String errorStr; + @ApiModelProperty(position = 7, value = "String value representing the type of message routing", example = "Success") + protected String relationType; + @ApiModelProperty(position = 8, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity") + protected String dataSearch; + @ApiModelProperty(position = 9, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName") + protected String metadataSearch; - public void setIsError(boolean isError) { - this.isError = isError; + @Override + public EventType getEventType() { + return EventType.DEBUG_RULE_NODE; } @Override - public boolean hasFilterForJsonBody() { - return !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(server) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch) - || !StringUtils.isEmpty(entityName) || !StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(msgType) || !StringUtils.isEmpty(errorStr) || isError; + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(entityId) + || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) || + !StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch); } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java new file mode 100644 index 0000000000..0e5de8263f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class StatisticsEvent extends Event { + + private static final long serialVersionUID = 6683733979448910631L; + + @Builder + private StatisticsEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, long messagesProcessed, long errorsOccurred) { + super(tenantId, entityId, serviceId, id, ts); + this.messagesProcessed = messagesProcessed; + this.errorsOccurred = errorsOccurred; + } + + @Getter + private final long messagesProcessed; + @Getter + private final long errorsOccurred; + + @Override + public EventType getType() { + return EventType.STATS; + } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + return eventInfo; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java index b132ff02da..598d750c0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java @@ -27,9 +27,13 @@ public class StatisticsEventFilter implements EventFilter { @ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") protected String server; @ApiModelProperty(position = 2, value = "The minimum number of successfully processed messages", example = "25") - protected Integer messagesProcessed; - @ApiModelProperty(position = 3, value = "The minimum number of errors occurred during messages processing", example = "30") - protected Integer errorsOccurred; + protected Integer minMessagesProcessed; + @ApiModelProperty(position = 3, value = "The maximum number of successfully processed messages", example = "250") + protected Integer maxMessagesProcessed; + @ApiModelProperty(position = 4, value = "The minimum number of errors occurred during messages processing", example = "30") + protected Integer minErrorsOccurred; + @ApiModelProperty(position = 5, value = "The maximum number of errors occurred during messages processing", example = "300") + protected Integer maxErrorsOccurred; @Override public EventType getEventType() { @@ -37,7 +41,9 @@ public class StatisticsEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { - return !StringUtils.isEmpty(server) || (messagesProcessed != null && messagesProcessed > 0) || (errorsOccurred != null && errorsOccurred > 0); + public boolean isNotEmpty() { + return !StringUtils.isEmpty(server) + || (minMessagesProcessed != null && minMessagesProcessed > 0) || (minErrorsOccurred != null && minErrorsOccurred > 0) + || (maxMessagesProcessed != null && maxMessagesProcessed > 0) || (maxErrorsOccurred != null && maxErrorsOccurred > 0); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 3937bbd9c3..88d68fa574 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -15,31 +15,41 @@ */ package org.thingsboard.server.dao.event; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; -import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.stream.Collectors; @Service @Slf4j public class BaseEventService implements EventService { + @Value("${sql.ttl.events.events_ttl:0}") + private long ttlInSec; + @Value("${sql.ttl.events.debug_events_ttl:604800}") + private long debugTtlInSec; + @Value("${event.debug.max-symbols:4096}") private int maxDebugEventSymbols; @@ -57,78 +67,85 @@ public class BaseEventService implements EventService { } private void checkAndTruncateDebugEvent(Event event) { - if (event.getType().startsWith("DEBUG") && event.getBody() != null && event.getBody().has("data")) { - String dataStr = event.getBody().get("data").asText(); - int length = dataStr.length(); + switch (event.getType()) { + case DEBUG_RULE_NODE: + RuleNodeDebugEvent rnEvent = (RuleNodeDebugEvent) event; + truncateField(rnEvent, RuleNodeDebugEvent::getData, RuleNodeDebugEvent::setData); + truncateField(rnEvent, RuleNodeDebugEvent::getMetadata, RuleNodeDebugEvent::setMetadata); + truncateField(rnEvent, RuleNodeDebugEvent::getError, RuleNodeDebugEvent::setError); + break; + case DEBUG_RULE_CHAIN: + RuleChainDebugEvent rcEvent = (RuleChainDebugEvent) event; + truncateField(rcEvent, RuleChainDebugEvent::getMessage, RuleChainDebugEvent::setMessage); + truncateField(rcEvent, RuleChainDebugEvent::getError, RuleChainDebugEvent::setError); + break; + case LC_EVENT: + LifecycleEvent lcEvent = (LifecycleEvent) event; + truncateField(lcEvent, LifecycleEvent::getError, LifecycleEvent::setError); + break; + case ERROR: + ErrorEvent eEvent = (ErrorEvent) event; + truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); + break; + } + } + + private void truncateField(T event, Function getter, BiConsumer setter) { + var str = getter.apply(event); + if (StringUtils.isNotEmpty(str)) { + var length = str.length(); if (length > maxDebugEventSymbols) { - ((ObjectNode) event.getBody()).put("data", dataStr.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]"); - log.trace("[{}] Event was truncated: {}", event.getId(), dataStr); + setter.accept(event, str.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]"); } } } @Override - public Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) { - if (tenantId == null) { - throw new DataValidationException("Tenant id should be specified!."); - } - if (entityId == null) { - throw new DataValidationException("Entity id should be specified!."); - } - if (StringUtils.isEmpty(eventType)) { - throw new DataValidationException("Event type should be specified!."); - } - if (StringUtils.isEmpty(eventUid)) { - throw new DataValidationException("Event uid should be specified!."); - } - Event event = eventDao.findEvent(tenantId.getId(), entityId, eventType, eventUid); - return event != null ? Optional.of(event) : Optional.empty(); + public PageData findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink) { + return convert(entityId.getEntityType(), eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, pageLink)); } @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink) { - return eventDao.findEvents(tenantId.getId(), entityId, pageLink); + public List findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit) { + return convert(entityId.getEntityType(), eventDao.findLatestEvents(tenantId.getId(), entityId.getId(), eventType, limit)); } @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { - return eventDao.findEvents(tenantId.getId(), entityId, eventType, pageLink); + public PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { + return convert(entityId.getEntityType(), eventDao.findEventByFilter(tenantId.getId(), entityId.getId(), eventFilter, pageLink)); } @Override - public List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) { - return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit); + public void removeEvents(TenantId tenantId, EntityId entityId) { + removeEvents(tenantId, entityId, null, null, null); } @Override - public PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { - return eventDao.findEventByFilter(tenantId.getId(), entityId, eventFilter, pageLink); + public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) { + if (eventFilter == null) { + eventDao.removeEvents(tenantId.getId(), entityId.getId(), startTime, endTime); + } else { + eventDao.removeEvents(tenantId.getId(), entityId.getId(), eventFilter, startTime, endTime); + } } @Override - public void removeEvents(TenantId tenantId, EntityId entityId) { - removeEvents(tenantId, entityId, null, null, null); + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) { + eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs, cleanupDb); } @Override - public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) { - TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime); - PageData eventsPageData; - do { - if (eventFilter == null) { - eventsPageData = findEvents(tenantId, entityId, eventsPageLink); - } else { - eventsPageData = findEventsByFilter(tenantId, entityId, eventFilter, eventsPageLink); - } + public void migrateEvents() { + eventDao.migrateEvents(ttlInSec > 0 ? System.currentTimeMillis() - ttlInSec : 0, debugTtlInSec > 0 ? System.currentTimeMillis() - debugTtlInSec : 0); + } - eventDao.removeAllByIds(eventsPageData.getData().stream() - .map(IdBased::getUuidId) - .collect(Collectors.toList())); - } while (eventsPageData.hasNext()); + private PageData convert(EntityType entityType, PageData pd) { + return new PageData<>(pd.getData() == null ? null : + pd.getData().stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()) + , pd.getTotalPages(), pd.getTotalElements(), pd.hasNext()); } - @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { - eventDao.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + private List convert(EntityType entityType, List list) { + return list == null ? null : list.stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 86eb0d91ec..7d26fabe62 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -16,12 +16,11 @@ package org.thingsboard.server.dao.event; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.dao.Dao; import java.util.List; import java.util.UUID; @@ -29,7 +28,7 @@ import java.util.UUID; /** * The Interface EventDao. */ -public interface EventDao extends Dao { +public interface EventDao { /** * Save or update event object async @@ -39,27 +38,6 @@ public interface EventDao extends Dao { */ ListenableFuture saveAsync(Event event); - /** - * Find event by tenantId, entityId and eventUid. - * - * @param tenantId the tenantId - * @param entityId the entityId - * @param eventType the eventType - * @param eventUid the eventUid - * @return the event - */ - Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid); - - /** - * Find events by tenantId, entityId and pageLink. - * - * @param tenantId the tenantId - * @param entityId the entityId - * @param pageLink the pageLink - * @return the event list - */ - PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink); - /** * Find events by tenantId, entityId, eventType and pageLink. * @@ -69,9 +47,9 @@ public interface EventDao extends Dao { * @param pageLink the pageLink * @return the event list */ - PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink); + PageData findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink); - PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); + PageData findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink); /** * Find latest events by tenantId, entityId and eventType. @@ -82,14 +60,37 @@ public interface EventDao extends Dao { * @param limit the limit * @return the event list */ - List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit); + List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit); /** * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. - * @param regularEventStartTs the start time of the interval to use to delete non debug events - * @param regularEventEndTs the end time of the interval to use to delete non debug events - * @param debugEventStartTs the start time of the interval to use to delete debug events - * @param debugEventEndTs the end time of the interval to use to delete debug events + * @param regularEventExpTs the expiration time of the regular events + * @param debugEventExpTs the expiration time of the debug events + * @param cleanupDb + */ + void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb); + + /** + * Removes all events for the specified entity and time interval + * + * @param tenantId + * @param entityId + * @param startTime + * @param endTime */ - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime); + + /** + * + * Removes all events for the specified entity, event filter and time interval + * + * @param tenantId + * @param entityId + * @param eventFilter + * @param startTime + * @param endTime + */ + void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime); + + void migrateEvents(long regularEventTs, long debugEventTs); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 0b4a65f43c..171417402d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -368,16 +368,35 @@ public class ModelConstants { /** * Cassandra event constants. */ - public static final String EVENT_COLUMN_FAMILY_NAME = "event"; + public static final String ERROR_EVENT_TABLE_NAME = "error_event"; + public static final String LC_EVENT_TABLE_NAME = "lc_event"; + public static final String STATS_EVENT_TABLE_NAME = "stats_event"; + public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; + public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; - public static final String EVENT_TYPE_PROPERTY = "event_type"; - public static final String EVENT_UID_PROPERTY = "event_uid"; - public static final String EVENT_ENTITY_TYPE_PROPERTY = ENTITY_TYPE_PROPERTY; + public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; public static final String EVENT_ENTITY_ID_PROPERTY = "entity_id"; public static final String EVENT_BODY_PROPERTY = "body"; - public static final String EVENT_BY_TYPE_AND_ID_VIEW_NAME = "event_by_type_and_id"; - public static final String EVENT_BY_ID_VIEW_NAME = "event_by_id"; + public static final String EVENT_MESSAGES_PROCESSED_COLUMN_NAME = "e_messages_processed"; + public static final String EVENT_ERRORS_OCCURRED_COLUMN_NAME = "e_errors_occurred"; + + public static final String EVENT_METHOD_COLUMN_NAME = "e_method"; + + public static final String EVENT_TYPE_COLUMN_NAME = "e_type"; + public static final String EVENT_ERROR_COLUMN_NAME = "e_error"; + public static final String EVENT_SUCCESS_COLUMN_NAME = "e_success"; + + public static final String EVENT_ENTITY_ID_COLUMN_NAME = "e_entity_id"; + public static final String EVENT_ENTITY_TYPE_COLUMN_NAME = "e_entity_type"; + public static final String EVENT_MSG_ID_COLUMN_NAME = "e_msg_id"; + public static final String EVENT_MSG_TYPE_COLUMN_NAME = "e_msg_type"; + public static final String EVENT_DATA_TYPE_COLUMN_NAME = "e_data_type"; + public static final String EVENT_RELATION_TYPE_COLUMN_NAME = "e_relation_type"; + public static final String EVENT_DATA_COLUMN_NAME = "e_data"; + public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; + public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; public static final String DEBUG_MODE = "debug_mode"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 4da17d2c34..8e7474f96e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; - import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index f2d1970238..893c9d549c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseSqlEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java new file mode 100644 index 0000000000..2175c97c94 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.ERROR_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METHOD_COLUMN_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = ERROR_EVENT_TABLE_NAME) +@NoArgsConstructor +public class ErrorEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_METHOD_COLUMN_NAME) + private String method; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public ErrorEventEntity(ErrorEvent event) { + super(event); + this.method = event.getMethod(); + this.error = event.getError(); + } + + @Override + public ErrorEvent toData() { + return ErrorEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .method(method) + .error(error) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index b452ec5bda..0e78d22600 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -15,103 +15,77 @@ */ package org.thingsboard.server.dao.model.sql; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import org.hibernate.annotations.Type; -import org.hibernate.annotations.TypeDef; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EventId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; -import org.thingsboard.server.dao.util.mapping.JsonStringType; +import org.thingsboard.server.dao.model.ModelConstants; import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Table; +import javax.persistence.Id; +import javax.persistence.MappedSuperclass; import java.util.UUID; -import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SERVICE_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data -@EqualsAndHashCode(callSuper = true) -@Entity -@TypeDef(name = "json", typeClass = JsonStringType.class) -@Table(name = EVENT_COLUMN_FAMILY_NAME) @NoArgsConstructor -public class EventEntity extends BaseSqlEntity implements BaseEntity { +@MappedSuperclass +public abstract class EventEntity implements BaseEntity { - @Column(name = EVENT_TENANT_ID_PROPERTY) - private UUID tenantId; + @Id + @Column(name = ModelConstants.ID_PROPERTY, columnDefinition = "uuid") + protected UUID id; - @Enumerated(EnumType.STRING) - @Column(name = EVENT_ENTITY_TYPE_PROPERTY) - private EntityType entityType; + @Column(name = EVENT_TENANT_ID_PROPERTY, columnDefinition = "uuid") + protected UUID tenantId; - @Column(name = EVENT_ENTITY_ID_PROPERTY) - private UUID entityId; + @Column(name = EVENT_ENTITY_ID_PROPERTY, columnDefinition = "uuid") + protected UUID entityId; - @Column(name = EVENT_TYPE_PROPERTY) - private String eventType; - - @Column(name = EVENT_UID_PROPERTY) - private String eventUid; - - @Type(type = "json") - @Column(name = EVENT_BODY_PROPERTY) - private JsonNode body; + @Column(name = EVENT_SERVICE_ID_PROPERTY) + protected String serviceId; @Column(name = TS_COLUMN) - private long ts; + protected long ts; + + public EventEntity(UUID id, UUID tenantId, UUID entityId, String serviceId, long ts) { + this.id = id; + this.tenantId = tenantId; + this.entityId = entityId; + this.serviceId = serviceId; + this.ts = ts; + } public EventEntity(Event event) { - if (event.getId() != null) { - this.setUuid(event.getId().getId()); - this.ts = getTs(event.getId().getId()); - } else { - this.ts = System.currentTimeMillis(); - } - this.setCreatedTime(event.getCreatedTime()); - if (event.getTenantId() != null) { - this.tenantId = event.getTenantId().getId(); - } - if (event.getEntityId() != null) { - this.entityType = event.getEntityId().getEntityType(); - this.entityId = event.getEntityId().getId(); - } - this.eventType = event.getType(); - this.eventUid = event.getUid(); - this.body = event.getBody(); + this.id = event.getId().getId(); + this.tenantId = event.getTenantId().getId(); + this.entityId = event.getEntityId(); + this.serviceId = event.getServiceId(); + this.ts = event.getCreatedTime(); } + @Override + public UUID getUuid() { + return id; + } @Override - public Event toData() { - Event event = new Event(new EventId(this.getUuid())); - event.setCreatedTime(createdTime); - event.setTenantId(TenantId.fromUUID(tenantId)); - event.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - event.setBody(body); - event.setType(eventType); - event.setUid(eventUid); - return event; + public void setUuid(UUID id) { + this.id = id; } - private static long getTs(UUID uuid) { - return (uuid.timestamp() - EPOCH_DIFF) / 10000; + @Override + public long getCreatedTime() { + return ts; } + + @Override + public void setCreatedTime(long createdTime) { + ts = createdTime; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java new file mode 100644 index 0000000000..874cc4ea00 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = LC_EVENT_TABLE_NAME) +@NoArgsConstructor +public class LifecycleEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_TYPE_COLUMN_NAME) + private String eventType; + @Column(name = EVENT_SUCCESS_COLUMN_NAME) + private boolean success; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public LifecycleEventEntity(LifecycleEvent event) { + super(event); + this.eventType = event.getLcEventType(); + this.success = event.isSuccess(); + this.error = event.getError(); + } + + @Override + public LifecycleEvent toData() { + return LifecycleEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .lcEventType(eventType) + .success(success) + .error(error) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java new file mode 100644 index 0000000000..7968521f62 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.RULE_CHAIN_DEBUG_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = RULE_CHAIN_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class RuleChainDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_MESSAGE_COLUMN_NAME) + private String message; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public RuleChainDebugEventEntity(RuleChainDebugEvent event) { + super(event); + this.message = event.getMessage(); + this.error = event.getError(); + } + + @Override + public RuleChainDebugEvent toData() { + return RuleChainDebugEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .message(message) + .error(error).build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java new file mode 100644 index 0000000000..6e75515b14 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METADATA_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_RELATION_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.RULE_NODE_DEBUG_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = RULE_NODE_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class RuleNodeDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_TYPE_COLUMN_NAME) + private String eventType; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_DATA_TYPE_COLUMN_NAME) + private String dataType; + @Column(name = EVENT_RELATION_TYPE_COLUMN_NAME) + private String relationType; + @Column(name = EVENT_DATA_COLUMN_NAME) + private String data; + @Column(name = EVENT_METADATA_COLUMN_NAME) + private String metadata; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public RuleNodeDebugEventEntity(RuleNodeDebugEvent event) { + super(event); + this.eventType = event.getEventType(); + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.dataType = event.getDataType(); + this.relationType = event.getRelationType(); + this.data = event.getData(); + this.metadata = event.getMetadata(); + this.error = event.getError(); + } + + @Override + public RuleNodeDebugEvent toData() { + var builder = RuleNodeDebugEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .eventType(eventType) + .msgId(msgId) + .msgType(msgType) + .dataType(dataType) + .relationType(relationType) + .data(data) + .metadata(metadata) + .error(error); + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java new file mode 100644 index 0000000000..9e151a1c24 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERRORS_OCCURRED_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGES_PROCESSED_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = STATS_EVENT_TABLE_NAME) +@NoArgsConstructor +public class StatisticsEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_MESSAGES_PROCESSED_COLUMN_NAME) + private long messagesProcessed; + @Column(name = EVENT_ERRORS_OCCURRED_COLUMN_NAME) + private long errorsOccurred; + + public StatisticsEventEntity(StatisticsEvent event) { + super(event); + this.messagesProcessed = event.getMessagesProcessed(); + this.errorsOccurred = event.getErrorsOccurred(); + } + + @Override + public StatisticsEvent toData() { + return StatisticsEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .messagesProcessed(messagesProcessed) + .errorsOccurred(errorsOccurred) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java index ae7b7542a4..8ad541ebac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EventDataValidator.java @@ -17,7 +17,7 @@ package org.thingsboard.server.dao.service.validator; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; @@ -27,14 +27,14 @@ public class EventDataValidator extends DataValidator { @Override protected void validateDataImpl(TenantId tenantId, Event event) { + if (event.getTenantId() == null) { + throw new DataValidationException("Tenant id should be specified!."); + } if (event.getEntityId() == null) { throw new DataValidationException("Entity id should be specified!."); } - if (StringUtils.isEmpty(event.getType())) { - throw new DataValidationException("Event type should be specified!."); - } - if (event.getBody() == null) { - throw new DataValidationException("Event body should be specified!."); + if (StringUtils.isEmpty(event.getServiceId())) { + throw new DataValidationException("Service id should be specified!."); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java new file mode 100644 index 0000000000..43b4d0c91d --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java @@ -0,0 +1,114 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.dao.model.sql.ErrorEventEntity; +import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; + +import java.util.List; +import java.util.UUID; + + +public interface ErrorEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM error_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM ErrorEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("method") String method, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM ErrorEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("method") String method, + @Param("error") String error); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java index 757d69f794..dd982727a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.sql.event; public interface EventCleanupRepository { - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void cleanupEvents(long eventExpTime, boolean debug); + void migrateEvents(long regularEventTs, long debugEventTs); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 6234db58da..85220ddf9f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -24,12 +24,24 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.server.dao.model.sql.EventEntity; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import javax.annotation.PostConstruct; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Types; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; @Repository @Transactional @@ -39,10 +51,7 @@ public class EventInsertRepository { private static final String EMPTY_STR = ""; - private static final String INSERT = - "INSERT INTO event (id, created_time, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) " + - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " + - "ON CONFLICT DO NOTHING;"; + private final Map insertStmtMap = new ConcurrentHashMap<>(); @Autowired protected JdbcTemplate jdbcTemplate; @@ -53,34 +62,172 @@ public class EventInsertRepository { @Value("${sql.remove_null_chars:true}") private boolean removeNullChars; - protected void save(List entities) { + @PostConstruct + public void init() { + insertStmtMap.put(EventType.ERROR, "INSERT INTO " + EventType.ERROR.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_method, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.LC_EVENT, "INSERT INTO " + EventType.LC_EVENT.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_type, e_success, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.STATS, "INSERT INTO " + EventType.STATS.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_messages_processed, e_errors_occurred) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_RULE_NODE, "INSERT INTO " + EventType.DEBUG_RULE_NODE.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_type, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_data_type, e_relation_type, e_data, e_metadata, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + } + + protected void save(List entities) { + Map> eventsByType = entities.stream().collect(Collectors.groupingBy(Event::getType, Collectors.toList())); transactionTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus status) { - jdbcTemplate.batchUpdate(INSERT, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - EventEntity event = entities.get(i); - ps.setObject(1, event.getId()); - ps.setLong(2, event.getCreatedTime()); - ps.setString(3, replaceNullChars(event.getBody().toString())); - ps.setObject(4, event.getEntityId()); - ps.setString(5, event.getEntityType().name()); - ps.setString(6, event.getEventType()); - ps.setString(7, event.getEventUid()); - ps.setObject(8, event.getTenantId()); - ps.setLong(9, event.getTs()); - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); + for (var entry : eventsByType.entrySet()) { + jdbcTemplate.batchUpdate(insertStmtMap.get(entry.getKey()), getStatementSetter(entry.getKey(), entry.getValue())); + } } }); } + private BatchPreparedStatementSetter getStatementSetter(EventType eventType, List events) { + switch (eventType) { + case ERROR: + return getErrorEventSetter(events); + case LC_EVENT: + return getLcEventSetter(events); + case STATS: + return getStatsEventSetter(events); + case DEBUG_RULE_NODE: + return getRuleNodeEventSetter(events); + case DEBUG_RULE_CHAIN: + return getRuleChainEventSetter(events); + default: + throw new RuntimeException(eventType + " support is not implemented!"); + } + } + + private BatchPreparedStatementSetter getErrorEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ErrorEvent event = (ErrorEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getMethod()); + safePutString(ps, 7, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getLcEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + LifecycleEvent event = (LifecycleEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getLcEventType()); + ps.setBoolean(7, event.isSuccess()); + safePutString(ps, 8, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getStatsEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + StatisticsEvent event = (StatisticsEvent) events.get(i); + setCommonEventFields(ps, event); + ps.setLong(6, event.getMessagesProcessed()); + ps.setLong(7, event.getErrorsOccurred()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getRuleNodeEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RuleNodeDebugEvent event = (RuleNodeDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getEventType()); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getDataType()); + safePutString(ps, 12, event.getRelationType()); + safePutString(ps, 13, event.getData()); + safePutString(ps, 14, event.getMetadata()); + safePutString(ps, 15, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getRuleChainEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RuleChainDebugEvent event = (RuleChainDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getMessage()); + safePutString(ps, 7, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { + if (value != null) { + ps.setString(parameterIdx, replaceNullChars(value)); + } else { + ps.setNull(parameterIdx, Types.VARCHAR); + } + } + + void safePutUUID(PreparedStatement ps, int parameterIdx, UUID value) throws SQLException { + if (value != null) { + ps.setObject(parameterIdx, value); + } else { + ps.setNull(parameterIdx, Types.OTHER); + } + } + + private void setCommonEventFields(PreparedStatement ps, Event event) throws SQLException { + ps.setObject(1, event.getId().getId()); + ps.setObject(2, event.getTenantId().getId()); + ps.setLong(3, event.getCreatedTime()); + ps.setObject(4, event.getEntityId()); + ps.setString(5, event.getServiceId()); + } + private String replaceNullChars(String strValue) { if (removeNullChars && strValue != null) { return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java new file mode 100644 index 0000000000..ef49ee9521 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.event.EventType; + +import javax.annotation.PostConstruct; +import java.util.concurrent.TimeUnit; + +@Component +public class EventPartitionConfiguration { + + @Getter + @Value("${sql.events.partition_size:168}") + private int regularPartitionSizeInHours; + @Getter + @Value("${sql.events.debug_partition_size:1}") + private int debugPartitionSizeInHours; + + private long regularPartitionSizeInMs; + private long debugPartitionSizeInMs; + + @PostConstruct + public void init() { + regularPartitionSizeInMs = TimeUnit.HOURS.toMillis(regularPartitionSizeInHours); + debugPartitionSizeInMs = TimeUnit.HOURS.toMillis(debugPartitionSizeInHours); + } + + public long getPartitionSizeInMs(EventType eventType) { + return eventType.isDebug() ? debugPartitionSizeInMs : regularPartitionSizeInMs; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index e36b82f1e8..19865cdc8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -21,224 +21,19 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.dao.model.sql.EventEntity; import java.util.List; import java.util.UUID; -/** - * Created by Valerii Sosliuk on 5/3/2017. - */ -public interface EventRepository extends JpaRepository { - - EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, - EntityType entityType, - UUID entityId, - String eventType, - String eventUid); - - EventEntity findByTenantIdAndEntityTypeAndEntityId(UUID tenantId, - EntityType entityType, - UUID entityId); - - @Query("SELECT e FROM EventEntity e WHERE e.tenantId = :tenantId AND e.entityType = :entityType " + - "AND e.entityId = :entityId AND e.eventType = :eventType ORDER BY e.eventType DESC, e.id DESC") - List findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( - @Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("eventType") String eventType, - Pageable pageable); - - @Query("SELECT e FROM EventEntity e WHERE " + - "e.tenantId = :tenantId " + - "AND e.entityType = :entityType AND e.entityId = :entityId " + - "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR e.createdTime <= :endTime) " + - "AND LOWER(e.eventType) LIKE LOWER(CONCAT('%', :textSearch, '%'))" - ) - Page findEventsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("textSearch") String textSearch, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); - - @Query("SELECT e FROM EventEntity e WHERE " + - "e.tenantId = :tenantId " + - "AND e.entityType = :entityType AND e.entityId = :entityId " + - "AND e.eventType = :eventType " + - "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR e.createdTime <= :endTime)" - ) - Page findEventsByTenantIdAndEntityIdAndEventType(@Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("eventType") String eventType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); - - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = :eventType " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + - "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + - "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + - "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + - "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + - "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + - "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + - "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%')) ", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = :eventType " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + - "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + - "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + - "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + - "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + - "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + - "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + - "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%'))" - ) - Page findDebugRuleNodeEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("eventType") String eventType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - @Param("type") String type, - @Param("server") String server, - @Param("entityName") String entityName, - @Param("relationType") String relationType, - @Param("bodyEntityId") String bodyEntityId, - @Param("msgType") String msgType, - @Param("isError") boolean isError, - @Param("error") String error, - @Param("data") String data, - @Param("metadata") String metadata, - Pageable pageable); +public interface EventRepository, V extends Event> { - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'ERROR' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'ERROR' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))") - Page findErrorEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("method") String method, - @Param("error") String error, - Pageable pageable); + List findLatestEvents(UUID tenantId, UUID entityId, int limit); - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " + - "AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))" - , - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " + - "AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))" - ) - Page findLifeCycleEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("event") String event, - @Param("statusFilterEnabled") boolean statusFilterEnabled, - @Param("statusFilter") boolean statusFilter, - @Param("error") String error, - Pageable pageable); + Page findEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime, Pageable pageable); - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'STATS' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + - "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + - "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ") - Page findStatisticsEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("messagesProcessed") Integer messagesProcessed, - @Param("errorsOccurred") Integer errorsOccurred, - Pageable pageable); + void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 2b409006d2..bfea45307f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -18,19 +18,20 @@ package org.thingsboard.server.dao.sql.event; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.event.DebugEvent; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.RuleChainDebugEventFilter; +import org.thingsboard.server.common.data.event.RuleNodeDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifeCycleEventFilter; import org.thingsboard.server.common.data.event.StatisticsEventFilter; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; @@ -38,32 +39,47 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.model.sql.EventEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.timeseries.SqlPartition; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; - /** * Created by Valerii Sosliuk on 5/3/2017. */ @Slf4j @Component -public class JpaBaseEventDao extends JpaAbstractDao implements EventDao { +public class JpaBaseEventDao implements EventDao { + + private final Map> partitionsByEventType = new ConcurrentHashMap<>(); + private static final ReentrantLock partitionCreationLock = new ReentrantLock(); + + @Autowired + private EventPartitionConfiguration partitionConfiguration; + + @Autowired + private SqlPartitioningRepository partitioningRepository; - private final UUID systemTenantId = NULL_UUID; + @Autowired + private LifecycleEventRepository lcEventRepository; + + @Autowired + private StatisticsEventRepository statsEventRepository; @Autowired - private EventRepository eventRepository; + private ErrorEventRepository errorEventRepository; @Autowired private EventInsertRepository eventInsertRepository; @@ -71,15 +87,11 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen @Autowired private EventCleanupRepository eventCleanupRepository; - @Override - protected Class getEntityClass() { - return EventEntity.class; - } + @Autowired + private RuleNodeDebugEventRepository ruleNodeDebugEventRepository; - @Override - protected JpaRepository getRepository() { - return eventRepository; - } + @Autowired + private RuleChainDebugEventRepository ruleChainDebugEventRepository; @Autowired ScheduledLogExecutorComponent logExecutor; @@ -102,10 +114,15 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen @Value("${sql.batch_sort:false}") private boolean batchSortEnabled; - private TbSqlBlockingQueueWrapper queue; + private TbSqlBlockingQueueWrapper queue; + + private final Map> repositories = new ConcurrentHashMap<>(); @PostConstruct private void init() { + for (EventType eventType : EventType.values()) { + partitionsByEventType.put(eventType, new ConcurrentHashMap<>()); + } TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder() .logName("Events") .batchSize(batchSize) @@ -114,11 +131,14 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen .statsNamePrefix("events") .batchSortEnabled(batchSortEnabled) .build(); - Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); + Function hashcodeFunction = entity -> Objects.hash(super.hashCode(), entity.getTenantId(), entity.getEntityId()); queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory); - queue.init(logExecutor, v -> eventInsertRepository.save(v), - Comparator.comparing((EventEntity eventEntity) -> eventEntity.getTs()) - ); + queue.init(logExecutor, v -> eventInsertRepository.save(v), Comparator.comparing(Event::getCreatedTime)); + repositories.put(EventType.LC_EVENT, lcEventRepository); + repositories.put(EventType.STATS, statsEventRepository); + repositories.put(EventType.ERROR, errorEventRepository); + repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); + repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); } @PreDestroy @@ -143,117 +163,153 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen event.setCreatedTime(System.currentTimeMillis()); } } - if (StringUtils.isEmpty(event.getUid())) { - event.setUid(event.getId().toString()); - } - - return save(new EventEntity(event)); + savePartitionIfNotExist(event); + return queue.add(event); } - private ListenableFuture save(EventEntity entity) { - log.debug("Save event [{}] ", entity); - if (entity.getTenantId() == null) { - log.trace("Save system event with predefined id {}", systemTenantId); - entity.setTenantId(systemTenantId); - } - if (entity.getUuid() == null) { - entity.setUuid(Uuids.timeBased()); + private void savePartitionIfNotExist(Event event) { + EventType type = event.getType(); + var partitionsMap = partitionsByEventType.get(type); + var partitionDuration = partitionConfiguration.getPartitionSizeInMs(type); + long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % partitionDuration); + if (partitionsMap.get(partitionStartTs) == null) { + savePartition(partitionsMap, new SqlPartition(type.getTable(), partitionStartTs, partitionStartTs + partitionDuration, Long.toString(partitionStartTs))); } - if (StringUtils.isEmpty(entity.getEventUid())) { - entity.setEventUid(entity.getUuid().toString()); - } - return addToQueue(entity); } - private ListenableFuture addToQueue(EventEntity entity) { - return queue.add(entity); + private void savePartition(Map partitionsMap, SqlPartition sqlPartition) { + if (!partitionsMap.containsKey(sqlPartition.getStart())) { + partitionCreationLock.lock(); + try { + log.trace("Saving partition: {}", sqlPartition); + partitioningRepository.save(sqlPartition); + log.trace("Adding partition to map: {}", sqlPartition); + partitionsMap.put(sqlPartition.getStart(), sqlPartition); + } catch (DataIntegrityViolationException ex) { + log.trace("Error occurred during partition save:", ex); + if (ex.getCause() instanceof ConstraintViolationException) { + log.warn("Saving partition [{}] rejected. Event data will save to the DEFAULT partition.", sqlPartition.getPartitionDate()); + partitionsMap.put(sqlPartition.getStart(), sqlPartition); + } else { + throw new RuntimeException(ex); + } + } finally { + partitionCreationLock.unlock(); + } + } } @Override - public Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid) { - return DaoUtil.getData(eventRepository.findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid( - tenantId, entityId.getEntityType(), entityId.getId(), eventType, eventUid)); + public PageData findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink) { + return DaoUtil.toPageData(getEventRepository(eventType).findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); } @Override - public PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository - .findEventsByTenantIdAndEntityId( - tenantId, - entityId.getEntityType(), - entityId.getId(), - Objects.toString(pageLink.getTextSearch(), ""), - pageLink.getStartTime(), - pageLink.getEndTime(), - DaoUtil.toPageable(pageLink))); + public PageData findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink) { + if (eventFilter.isNotEmpty()) { + switch (eventFilter.getEventType()) { + case DEBUG_RULE_NODE: + return findEventByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, pageLink); + case DEBUG_RULE_CHAIN: + return findEventByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, pageLink); + case LC_EVENT: + return findEventByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, pageLink); + case ERROR: + return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); + case STATS: + return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + default: + throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); + } + } else { + return findEvents(tenantId, entityId, eventFilter.getEventType(), pageLink); + } } @Override - public PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository - .findEventsByTenantIdAndEntityIdAndEventType( - tenantId, - entityId.getEntityType(), - entityId.getId(), - eventType, - pageLink.getStartTime(), - pageLink.getEndTime(), - DaoUtil.toPageable(pageLink))); + public void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime) { + log.debug("[{}][{}] Remove events [{}-{}] ", tenantId, entityId, startTime, endTime); + for (EventType eventType : EventType.values()) { + getEventRepository(eventType).removeEvents(tenantId, entityId, startTime, endTime); + } } @Override - public PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { - if (eventFilter.hasFilterForJsonBody()) { + public void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime) { + if (eventFilter.isNotEmpty()) { switch (eventFilter.getEventType()) { case DEBUG_RULE_NODE: + removeEventsByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, startTime, endTime); + break; case DEBUG_RULE_CHAIN: - return findEventByFilter(tenantId, entityId, (DebugEvent) eventFilter, pageLink); + removeEventsByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, startTime, endTime); + break; case LC_EVENT: - return findEventByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, pageLink); + removeEventsByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, startTime, endTime); + break; case ERROR: - return findEventByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, pageLink); + removeEventsByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, startTime, endTime); + break; case STATS: - return findEventByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, pageLink); + removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); + break; default: throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } } else { - return findEvents(tenantId, entityId, eventFilter.getEventType().name(), pageLink); + getEventRepository(eventFilter.getEventType()).removeEvents(tenantId, entityId, startTime, endTime); } } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, DebugEvent eventFilter, TimePageLink pageLink) { + @Override + public void migrateEvents(long regularEventTs, long debugEventTs) { + eventCleanupRepository.migrateEvents(regularEventTs, debugEventTs); + } + + private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, TimePageLink pageLink) { return DaoUtil.toPageData( - eventRepository.findDebugRuleNodeEvents( + ruleChainDebugEventRepository.findEvents( tenantId, - entityId.getId(), - entityId.getEntityType().name(), - eventFilter.getEventType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), - eventFilter.getMsgDirectionType(), + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), eventFilter.getServer(), - eventFilter.getEntityName(), - eventFilter.getRelationType(), - eventFilter.getEntityId(), - eventFilter.getMsgType(), + eventFilter.getMessage(), eventFilter.isError(), eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink))); + } + + private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + ruleNodeDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getMsgDirectionType(), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getRelationType(), eventFilter.getDataSearch(), eventFilter.getMetadataSearch(), + eventFilter.isError(), + eventFilter.getErrorStr(), DaoUtil.toPageable(pageLink))); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { + private PageData findEventByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { return DaoUtil.toPageData( - eventRepository.findErrorEvents( + errorEventRepository.findEvents( tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), eventFilter.getServer(), eventFilter.getMethod(), eventFilter.getErrorStr(), @@ -261,16 +317,15 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { + private PageData findEventByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); return DaoUtil.toPageData( - eventRepository.findLifeCycleEvents( + lcEventRepository.findEvents( tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), eventFilter.getServer(), eventFilter.getEvent(), statusFilterEnabled, @@ -280,44 +335,145 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { + private PageData findEventByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { return DaoUtil.toPageData( - eventRepository.findStatisticsEvents( + statsEventRepository.findEvents( tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), eventFilter.getServer(), - notNull(eventFilter.getMessagesProcessed()), - notNull(eventFilter.getErrorsOccurred()), + eventFilter.getMinMessagesProcessed(), + eventFilter.getMaxMessagesProcessed(), + eventFilter.getMinErrorsOccurred(), + eventFilter.getMaxErrorsOccurred(), DaoUtil.toPageable(pageLink)) ); } - @Override - public List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit) { - List latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { + ruleChainDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMessage(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + ruleNodeDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMsgDirectionType(), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getRelationType(), + eventFilter.getDataSearch(), + eventFilter.getMetadataSearch(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, Long startTime, Long endTime) { + errorEventRepository.removeEvents( tenantId, - entityId.getEntityType(), - entityId.getId(), - eventType, - PageRequest.of(0, limit)); - return DaoUtil.convertDataList(latest); + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMethod(), + eventFilter.getErrorStr()); + + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, Long startTime, Long endTime) { + boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); + boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); + lcEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getEvent(), + statusFilterEnabled, + statusFilter, + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, Long startTime, Long endTime) { + statsEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMinMessagesProcessed(), + eventFilter.getMaxMessagesProcessed(), + eventFilter.getMinErrorsOccurred(), + eventFilter.getMaxErrorsOccurred() + ); } @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { - log.info("Going to cleanup old events. Interval for regular events: [{}:{}], for debug events: [{}:{}]", regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); - eventCleanupRepository.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { + return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); } - private long notNull(Long value) { - return value != null ? value : 0; + @Override + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) { + if (regularEventExpTs > 0) { + log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs); + if (cleanupDb) { + eventCleanupRepository.cleanupEvents(regularEventExpTs, false); + } + cleanupPartitions(regularEventExpTs, false); + } + if (debugEventExpTs > 0) { + log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs); + if (cleanupDb) { + eventCleanupRepository.cleanupEvents(debugEventExpTs, true); + } + cleanupPartitions(debugEventExpTs, true); + } } - private int notNull(Integer value) { - return value != null ? value : 0; + private void cleanupPartitions(long expTime, boolean isDebug) { + for (EventType eventType : EventType.values()) { + if (eventType.isDebug() == isDebug) { + Map partitions = partitionsByEventType.get(eventType); + partitions.keySet().removeIf(startTs -> startTs + partitionConfiguration.getPartitionSizeInMs(eventType) < expTime); + } + } } + private void parseUUID(String src, String paramName) { + if (!StringUtils.isEmpty(src)) { + try { + UUID.fromString(src); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Failed to convert " + paramName + " to UUID!"); + } + } + } + + private EventRepository, ?> getEventRepository(EventType eventType) { + var repository = repositories.get(eventType); + if (repository == null) { + throw new RuntimeException("Event type: " + eventType + " is not supported!"); + } + return repository; + } + + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java new file mode 100644 index 0000000000..11215e4771 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface LifecycleEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM lc_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + + @Query("SELECT e FROM LifecycleEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String eventType, + @Param("statusFilterEnabled") boolean statusFilterEnabled, + @Param("statusFilter") boolean statusFilter, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM LifecycleEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String eventType, + @Param("statusFilterEnabled") boolean statusFilterEnabled, + @Param("statusFilter") boolean statusFilter, + @Param("error") String error); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java new file mode 100644 index 0000000000..eb8ba01cf0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; +import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity; + +import java.util.List; +import java.util.UUID; + + +public interface RuleChainDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM rule_chain_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM RuleChainDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("message") String message, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM RuleChainDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("message") String message, + @Param("isError") boolean isError, + @Param("error") String error); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java new file mode 100644 index 0000000000..3df9989674 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java @@ -0,0 +1,153 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.dao.model.sql.ErrorEventEntity; +import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; + +import java.util.List; +import java.util.UUID; + + +public interface RuleNodeDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String type, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("relationType") String relationType, + @Param("data") String data, + @Param("metadata") String metadata, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String type, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("relationType") String relationType, + @Param("data") String data, + @Param("metadata") String metadata, + @Param("isError") boolean isError, + @Param("error") String error); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index 6a17f7c740..53786c448b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -16,38 +16,175 @@ package org.thingsboard.server.dao.sql.event; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeUnit; + @Slf4j @Repository public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository { + private static final String SELECT_PARTITIONS_STMT = "SELECT tablename from pg_tables WHERE schemaname = 'public' and tablename like concat(?, '_%')"; + private static final int PSQL_VERSION_14 = 140000; + + @Autowired + private EventPartitionConfiguration partitionConfiguration; + + private volatile Integer currentServerVersion; + @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { + public void cleanupEvents(long eventExpTime, boolean debug) { + for (EventType eventType : EventType.values()) { + if (eventType.isDebug() == debug) { + cleanupEvents(eventType, eventExpTime); + } + } + } + + @Override + public void migrateEvents(long regularEventTs, long debugEventTs) { + regularEventTs = Math.max(regularEventTs, 1480982400000L); + debugEventTs = Math.max(debugEventTs, 1480982400000L); + + callMigrateFunctionByPartitions("regular", "migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours()); + callMigrateFunctionByPartitions("debug", "migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours()); + try (Connection connection = dataSource.getConnection(); - PreparedStatement stmt = connection.prepareStatement("call cleanup_events_by_ttl(?,?,?,?,?)")) { - stmt.setLong(1, regularEventStartTs); - stmt.setLong(2, regularEventEndTs); - stmt.setLong(3, debugEventStartTs); - stmt.setLong(4, debugEventEndTs); - stmt.setLong(5, 0); - stmt.setQueryTimeout((int) TimeUnit.HOURS.toSeconds(1)); + PreparedStatement dropFunction1 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_regular_events(bigint, bigint, int)"); + PreparedStatement dropFunction2 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_debug_events(bigint, bigint, int)"); + PreparedStatement dropTable = connection.prepareStatement("DROP TABLE IF EXISTS event")) { + dropFunction1.execute(); + dropFunction2.execute(); + dropTable.execute(); + } catch (SQLException e) { + log.error("SQLException occurred during drop of the `events` table", e); + throw new RuntimeException(e); + } + } + + private void callMigrateFunctionByPartitions(String logTag, String functionName, long startTs, int partitionSizeInHours) { + long currentTs = System.currentTimeMillis(); + var regularPartitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); + long numberOfPartitions = (currentTs - startTs) / regularPartitionStepInMs; + if (numberOfPartitions > 1000) { + log.error("Please adjust your {} events partitioning configuration. " + + "Configuration with partition size of {} hours and corresponding TTL will use {} (>1000) partitions which is not recommended!", + logTag, partitionSizeInHours, numberOfPartitions); + throw new RuntimeException("Please adjust your " + logTag + " events partitioning configuration. " + + "Configuration with partition size of " + partitionSizeInHours + " hours and corresponding TTL will use " + + +numberOfPartitions + " (>1000) partitions which is not recommended!"); + } + while (startTs < currentTs) { + var endTs = startTs + regularPartitionStepInMs; + log.info("Migrate {} events for time period: [{},{}]", logTag, startTs, endTs); + callMigrateFunction(functionName, startTs, startTs + regularPartitionStepInMs, partitionSizeInHours); + startTs = endTs; + } + log.info("Migrate {} events done.", logTag); + } + + private void callMigrateFunction(String functionName, long startTs, long endTs, int partitionSizeInHours) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("call " + functionName + "(?,?,?)")) { + stmt.setLong(1, startTs); + stmt.setLong(2, endTs); + stmt.setInt(3, partitionSizeInHours); stmt.execute(); - printWarnings(stmt); - try (ResultSet resultSet = stmt.getResultSet()){ - resultSet.next(); - log.info("Total events removed by TTL: [{}]", resultSet.getLong(1)); + } catch (SQLException e) { + if (e.getMessage() == null || !e.getMessage().contains("relation \"event\" does not exist")) { + log.error("[{}] SQLException occurred during execution of {} with parameters {} and {}", functionName, startTs, partitionSizeInHours, e); + throw new RuntimeException(e); + } + } + } + + private void cleanupEvents(EventType eventType, long eventExpTime) { + var partitionDuration = partitionConfiguration.getPartitionSizeInMs(eventType); + List partitions = fetchPartitions(eventType); + for (var partitionTs : partitions) { + var partitionEndTs = partitionTs + partitionDuration; + if (partitionEndTs < eventExpTime) { + log.info("[{}] Detaching expired partition: [{}-{}]", eventType, partitionTs, partitionEndTs); + if (detachAndDropPartition(eventType, partitionTs)) { + log.info("[{}] Detached expired partition: {}", eventType, partitionTs); + } + } else { + log.debug("[{}] Skip valid partition: {}", eventType, partitionTs); + } + } + } + + private List fetchPartitions(EventType eventType) { + List partitions = new ArrayList<>(); + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement(SELECT_PARTITIONS_STMT)) { + stmt.setString(1, eventType.getTable()); + stmt.execute(); + try (ResultSet resultSet = stmt.getResultSet()) { + while (resultSet.next()) { + String partitionTableName = resultSet.getString(1); + String partitionTsStr = partitionTableName.substring(eventType.getTable().length() + 1); + try { + partitions.add(Long.parseLong(partitionTsStr)); + } catch (NumberFormatException nfe) { + log.warn("Failed to parse table name: {}", partitionTableName); + } + } } } catch (SQLException e) { log.error("SQLException occurred during events TTL task execution ", e); } + return partitions; + } + + private boolean detachAndDropPartition(EventType eventType, long partitionTs) { + String tablePartition = eventType.getTable() + "_" + partitionTs; + String detachPsqlStmtStr = "ALTER TABLE " + eventType.getTable() + " DETACH PARTITION " + tablePartition; + if (getCurrentServerVersion() >= PSQL_VERSION_14) { + detachPsqlStmtStr += " CONCURRENTLY"; + } + + String dropStmtStr = "DROP TABLE " + tablePartition; + try (Connection connection = dataSource.getConnection(); + PreparedStatement detachStmt = connection.prepareStatement(detachPsqlStmtStr); + PreparedStatement dropStmt = connection.prepareStatement(dropStmtStr)) { + detachStmt.execute(); + dropStmt.execute(); + return true; + } catch (SQLException e) { + log.error("[{}] SQLException occurred during detach and drop of the partition: {}", eventType, partitionTs, e); + } + return false; + } + + private synchronized int getCurrentServerVersion() { + if (currentServerVersion == null) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement versionStmt = connection.prepareStatement("SELECT current_setting('server_version_num')")) { + versionStmt.execute(); + try (ResultSet resultSet = versionStmt.getResultSet()) { + while (resultSet.next()) { + currentServerVersion = resultSet.getInt(1); + } + } + } catch (SQLException e) { + log.warn("SQLException occurred during fetch of the server version", e); + } + if (currentServerVersion == null) { + currentServerVersion = 0; + } + } + return currentServerVersion; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java new file mode 100644 index 0000000000..3d2c4f4516 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java @@ -0,0 +1,120 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; + +import java.util.List; +import java.util.UUID; + +public interface StatisticsEventRepository extends EventRepository, JpaRepository { + + @Override + @Query(nativeQuery = true, value = "SELECT * FROM stats_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); + + @Query("SELECT e FROM StatisticsEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" + , + countQuery = "SELECT count(*) FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("minMessagesProcessed") Integer minMessagesProcessed, + @Param("maxMessagesProcessed") Integer maxMessagesProcessed, + @Param("minErrorsOccurred") Integer minErrorsOccurred, + @Param("maxErrorsOccurred") Integer maxErrorsOccurred, + Pageable pageable); + + @Transactional + @Modifying + @Query("DELETE FROM StatisticsEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" + + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("minMessagesProcessed") Integer minMessagesProcessed, + @Param("maxMessagesProcessed") Integer maxMessagesProcessed, + @Param("minErrorsOccurred") Integer minErrorsOccurred, + @Param("maxErrorsOccurred") Integer maxErrorsOccurred); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java index 899a195538..805e5e4f4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java @@ -23,7 +23,6 @@ import org.thingsboard.server.dao.util.SqlTsDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -@SqlTsDao @Repository @Transactional public class SqlPartitioningRepository { @@ -32,8 +31,7 @@ public class SqlPartitioningRepository { private EntityManager entityManager; public void save(SqlPartition partition) { - entityManager.createNativeQuery(partition.getQuery()) - .executeUpdate(); + entityManager.createNativeQuery(partition.getQuery()).executeUpdate(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java index ce173e046b..64d1669eae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java @@ -132,7 +132,7 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao long partitionEndTs = toMills(localDateTimeEnd); ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); - savePartition(new SqlPartition(partitionStartTs, partitionEndTs, partitionDate)); + savePartition(new SqlPartition(SqlPartition.TS_KV, partitionStartTs, partitionEndTs, partitionDate)); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java index 5c47689b6d..dcdb335784 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java @@ -20,21 +20,21 @@ import lombok.Data; @Data public class SqlPartition { - private static final String TABLE_REGEX = "ts_kv_"; + public static final String TS_KV = "ts_kv"; private long start; private long end; private String partitionDate; private String query; - public SqlPartition(long start, long end, String partitionDate) { + public SqlPartition(String table, long start, long end, String partitionDate) { this.start = start; this.end = end; this.partitionDate = partitionDate; - this.query = createStatement(start, end, partitionDate); + this.query = createStatement(table, start, end, partitionDate); } - private String createStatement(long start, long end, String partitionDate) { - return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv FOR VALUES FROM (" + start + ") TO (" + end + ")"; + private String createStatement(String table, long start, long end, String partitionDate) { + return "CREATE TABLE IF NOT EXISTS " + table + "_" + partitionDate + " PARTITION OF " + table + " FOR VALUES FROM (" + start + ") TO (" + end + ")"; } } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index 7e15b419a8..d78257ad8b 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -21,18 +21,18 @@ -- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts -- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) -CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event - (ts DESC NULLS LAST) - WITH (FILLFACTOR=95); +CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); -COMMENT ON INDEX public.idx_event_ts - IS 'This index helps to delete events by TTL using timestamp'; +CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); -CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event - (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) - WITH (FILLFACTOR=95); +CREATE INDEX IF NOT EXISTS idx_stats_event_main + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_lc_event_main + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_error_event_main + ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); -COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a9b25a66f7..22f36a2bfb 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -323,18 +323,64 @@ CREATE TABLE IF NOT EXISTS device_credentials ( CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id) ); -CREATE TABLE IF NOT EXISTS event ( - id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY, - created_time bigint NOT NULL, - body varchar(10000000), - entity_id uuid, - entity_type varchar(255), - event_type varchar(255), - event_uid varchar(255), - tenant_id uuid, +CREATE TABLE IF NOT EXISTS rule_node_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , ts bigint NOT NULL, - CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) -); + entity_id uuid NOT NULL, + service_id varchar, + e_type varchar, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_data_type varchar, + e_relation_type varchar, + e_data varchar, + e_metadata varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_message varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS stats_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_messages_processed bigint NOT NULL, + e_errors_occurred bigint NOT NULL +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS lc_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_type varchar NOT NULL, + e_success boolean NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS error_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_method varchar NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS relation ( from_id uuid, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 964e6229f1..475f84a29a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -28,17 +28,20 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -185,17 +188,16 @@ public abstract class AbstractServiceTest { } - protected Event generateEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) throws IOException { + protected RuleNodeDebugEvent generateEvent(TenantId tenantId, EntityId entityId) throws IOException { if (tenantId == null) { tenantId = TenantId.fromUUID(Uuids.timeBased()); } - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(eventType); - event.setUid(eventUid); - event.setBody(readFromResource("TestJsonData.json")); - return event; + return RuleNodeDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId("server A") + .data(JacksonUtil.toString(readFromResource("TestJsonData.json"))) + .build(); } // // private ComponentDescriptor getOrCreateDescriptor(ComponentScope scope, ComponentType type, String clazz, String configurationDescriptorResource) throws IOException { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index f3f6dd8ff8..26e0df8c6d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -16,12 +16,13 @@ package org.thingsboard.server.dao.service.event; import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.apache.commons.lang3.time.DateFormatUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -33,10 +34,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.service.AbstractServiceTest; import java.text.ParseException; -import java.time.LocalDateTime; -import java.time.Month; -import java.time.ZoneOffset; -import java.util.Optional; +import java.util.List; import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; @@ -58,15 +56,14 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { @Test public void saveEvent() throws Exception { + TenantId tenantId = new TenantId(Uuids.timeBased()); DeviceId devId = new DeviceId(Uuids.timeBased()); - Event event = generateEvent(null, devId, "ALARM", Uuids.timeBased().toString()); + RuleNodeDebugEvent event = generateEvent(tenantId, devId); eventService.saveAsync(event).get(); - Optional loaded = eventService.findEvent(event.getTenantId(), event.getEntityId(), event.getType(), event.getUid()); - Assert.assertTrue(loaded.isPresent()); - Assert.assertNotNull(loaded.get()); - Assert.assertEquals(event.getEntityId(), loaded.get().getEntityId()); - Assert.assertEquals(event.getType(), loaded.get().getType()); - Assert.assertEquals(event.getBody(), loaded.get().getBody()); + List loaded = eventService.findLatestEvents(event.getTenantId(), devId, event.getType(), 1); + Assert.assertNotNull(loaded); + Assert.assertEquals(1, loaded.size()); + Assert.assertEquals(event.getData(), loaded.get(0).getBody().get("data").asText()); } @Test @@ -79,25 +76,24 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); - TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime); + TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts"), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, - timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 2); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); - Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); + Assert.assertEquals(2, events.getData().size()); + Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId()); + Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId()); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 1); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); + Assert.assertEquals(1, events.getData().size()); + Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId()); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true); } @Test @@ -110,30 +106,30 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); - TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); + TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts", SortOrder.Direction.DESC), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, - timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 2); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); - Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); + Assert.assertEquals(2, events.getData().size()); + Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId()); + Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId()); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 1); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); + Assert.assertEquals(1, events.getData().size()); + Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId()); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true); } private Event saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { - Event event = generateEvent(tenantId, entityId, DataConstants.STATS, null); - event.setId(new EventId(Uuids.startOf(time))); + RuleNodeDebugEvent event = generateEvent(tenantId, entityId); + event.setId(new EventId(Uuids.timeBased())); + event.setCreatedTime(time); eventService.saveAsync(event).get(); return event; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java index eb5f004714..b2091379b4 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java @@ -16,39 +16,27 @@ package org.thingsboard.server.dao.sql.event; import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; -import org.junit.After; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.AbstractJpaDaoTest; import org.thingsboard.server.dao.event.EventDao; -import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.thingsboard.server.common.data.DataConstants.ALARM; -import static org.thingsboard.server.common.data.DataConstants.STATS; -/** - * Created by Valerii Sosliuk on 5/5/2017. - */ @Slf4j public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { @@ -56,21 +44,21 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { private EventDao eventDao; UUID tenantId = Uuids.timeBased(); - @After - public void deleteEvents() { - List events = eventDao.find(TenantId.fromUUID(tenantId)); - for (Event event : events) { - eventDao.removeById(TenantId.fromUUID(tenantId), event.getUuidId()); - } - } @Test - public void findEvent() { + public void findEvent() throws InterruptedException, ExecutionException, TimeoutException { UUID entityId = Uuids.timeBased(); - Event savedEvent = eventDao.save(TenantId.fromUUID(tenantId), getEvent(entityId, tenantId, entityId)); - Event foundEvent = eventDao.findEvent(tenantId, new DeviceId(entityId), DataConstants.STATS, savedEvent.getUid()); - assertNotNull("Event expected to be not null", foundEvent); - assertEquals(savedEvent.getId(), foundEvent.getId()); + + Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId); + eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES); + Thread.sleep(2); + Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId); + eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES); + + List foundEvents = eventDao.findLatestEvents(tenantId, entityId, EventType.STATS, 1); + assertNotNull("Events expected to be not null", foundEvents); + assertEquals(1, foundEvents.size()); + assertEquals(event2, foundEvents.get(0)); } @Test @@ -78,108 +66,55 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { UUID entityId1 = Uuids.timeBased(); UUID entityId2 = Uuids.timeBased(); long startTime = System.currentTimeMillis(); - long endTime = createEventsTwoEntities(tenantId, entityId1, entityId2, 20); - TimePageLink pageLink1 = new TimePageLink(30); - PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink1); - assertEquals(10, events1.getData().size()); + Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId1); + eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES); + Thread.sleep(2); + Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId2); + eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES); - TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); - PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink2); - assertEquals(10, events2.getData().size()); + long endTime = System.currentTimeMillis(); - TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); - PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink3); - assertEquals(10, events3.getData().size()); - - TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); - PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(5, events4.getData().size()); - - pageLink4 = pageLink4.nextPageLink(); - PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(5, events5.getData().size()); - - pageLink4 = pageLink4.nextPageLink(); - PageData events6 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(0, events6.getData().size()); + PageData events1 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, new TimePageLink(30)); + assertEquals(1, events1.getData().size()); - } + PageData events2 = eventDao.findEvents(tenantId, entityId2, EventType.STATS, new TimePageLink(30)); + assertEquals(1, events2.getData().size()); - @Test - public void findEventsByEntityIdAndEventTypeAndPageLink() throws Exception { - UUID entityId1 = Uuids.timeBased(); - UUID entityId2 = Uuids.timeBased(); - long startTime = System.currentTimeMillis(); - long endTime = createEventsTwoEntitiesTwoTypes(tenantId, entityId1, entityId2, 20); + PageData events3 = eventDao.findEvents(tenantId, Uuids.timeBased(), EventType.STATS, new TimePageLink(30)); + assertEquals(0, events3.getData().size()); - TimePageLink pageLink1 = new TimePageLink(30); - PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink1); - assertEquals(5, events1.getData().size()); TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); - PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink2); - assertEquals(5, events2.getData().size()); + PageData events12 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink2); + assertEquals(1, events12.getData().size()); + assertEquals(event1, events12.getData().get(0)); TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); - PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink3); - assertEquals(5, events3.getData().size()); + PageData events13 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink3); + assertEquals(1, events13.getData().size()); + assertEquals(event1, events13.getData().get(0)); - TimePageLink pageLink4 = new TimePageLink(4, 0, "", null, startTime, endTime); - PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); - assertEquals(4, events4.getData().size()); + TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); + PageData events14 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4); + assertEquals(1, events14.getData().size()); + assertEquals(event1, events14.getData().get(0)); pageLink4 = pageLink4.nextPageLink(); - PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); - assertEquals(1, events5.getData().size()); - } - - private long createEventsTwoEntitiesTwoTypes(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { - for (int i = 0; i < count / 2; i++) { - String type = i % 2 == 0 ? STATS : ALARM; - UUID eventId1 = Uuids.timeBased(); - Event event1 = getEvent(eventId1, tenantId, entityId1, type); - eventDao.saveAsync(event1).get(); - UUID eventId2 = Uuids.timeBased(); - Event event2 = getEvent(eventId2, tenantId, entityId2, type); - eventDao.saveAsync(event2).get(); - } - return System.currentTimeMillis(); - } - - private long createEventsTwoEntities(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { - for (int i = 0; i < count / 2; i++) { - UUID eventId1 = Uuids.timeBased(); - Event event1 = getEvent(eventId1, tenantId, entityId1); - eventDao.saveAsync(event1).get(); - UUID eventId2 = Uuids.timeBased(); - Event event2 = getEvent(eventId2, tenantId, entityId2); - eventDao.saveAsync(event2).get(); - } - return System.currentTimeMillis(); - } + PageData events6 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4); + assertEquals(0, events6.getData().size()); - private Event getEvent(UUID eventId, UUID tenantId, UUID entityId, String type) { - Event event = getEvent(eventId, tenantId, entityId); - event.setType(type); - return event; } - private Event getEvent(UUID eventId, UUID tenantId, UUID entityId) { - Event event = new Event(); - event.setId(new EventId(eventId)); - event.setTenantId(TenantId.fromUUID(tenantId)); - EntityId deviceId = new DeviceId(entityId); - event.setEntityId(deviceId); - event.setUid(event.getId().getId().toString()); - event.setType(STATS); - ObjectMapper mapper = new ObjectMapper(); - try { - JsonNode jsonNode = mapper.readTree("{\"key\":\"value\"}"); - event.setBody(jsonNode); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - return event; + private Event getStatsEvent(UUID eventId, UUID tenantId, UUID entityId) { + StatisticsEvent.StatisticsEventBuilder event = StatisticsEvent.builder(); + event.id(eventId); + event.ts(System.currentTimeMillis()); + event.tenantId(new TenantId(tenantId)); + event.entityId(entityId); + event.serviceId("server A"); + event.messagesProcessed(1); + event.errorsOccurred(0); + return event.build(); } } diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index 4b3ea0a74d..5765153a6f 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -58,8 +58,6 @@ cassandra.query.ts_key_value_partitions_max_cache_size=100000 cassandra.query.ts_key_value_ttl=0 -cassandra.query.debug_events_ttl=604800 - cassandra.query.max_limit_per_request=1000 cassandra.query.buffer_size=100000 cassandra.query.concurrent_limit=1000 diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index a0f1d5dfa1..f2f446e534 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; @@ -1811,7 +1811,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } } - public PageData getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) { + public PageData getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) { Map params = new HashMap<>(); params.put("entityType", entityId.getEntityType().name()); params.put("entityId", entityId.getId().toString()); @@ -1823,12 +1823,12 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { baseURL + "/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&" + getTimeUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, - new ParameterizedTypeReference>() { + new ParameterizedTypeReference>() { }, params).getBody(); } - public PageData getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) { + public PageData getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) { Map params = new HashMap<>(); params.put("entityType", entityId.getEntityType().name()); params.put("entityId", entityId.getId().toString()); @@ -1839,7 +1839,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { baseURL + "/api/events/{entityType}/{entityId}?tenantId={tenantId}&" + getTimeUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, - new ParameterizedTypeReference>() { + new ParameterizedTypeReference>() { }, params).getBody(); } diff --git a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts index 14227d6c17..68b803e006 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts @@ -75,7 +75,7 @@ export class EventFilterPanelComponent { } isNumberFields(key: string): string { - return ['messagesProcessed', 'errorsOccurred'].includes(key) ? key : ''; + return ['minMessagesProcessed', 'maxMessagesProcessed', 'minErrorsOccurred', 'maxErrorsOccurred'].includes(key) ? key : ''; } selectorValues(key: string): string[] { diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 0aebf12bd5..997e4db414 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -225,7 +225,6 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_RULE_NODE: - case DebugEventType.DEBUG_RULE_CHAIN: this.columns[0].width = '100px'; this.columns.push( new EntityTableColumn('type', 'event.type', '40px', @@ -234,8 +233,8 @@ export class EventTableConfig extends EntityTableConfig { }), false, key => ({ padding: '0 12px 0 0' })), - new EntityTableColumn('entityName', 'event.entity-type', '100px', - (entity) => entity.body.entityName, entity => ({ + new EntityTableColumn('entityType', 'event.entity-type', '100px', + (entity) => entity.body.entityType, entity => ({ padding: '0 12px 0 0', }), false, key => ({ padding: '0 12px 0 0' @@ -289,6 +288,29 @@ export class EventTableConfig extends EntityTableConfig { '40px') ); break; + case DebugEventType.DEBUG_RULE_CHAIN: + this.columns[0].width = '100px'; + this.columns.push( + new EntityActionTableColumn('message', 'event.message', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.message ? entity.body.message.length > 0 : false, + onAction: ($event, entity) => this.showContent($event, entity.body.message, + 'event.message') + }, + '40px'), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '40px') + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -334,16 +356,18 @@ export class EventTableConfig extends EntityTableConfig { break; case EventType.STATS: this.filterColumns.push( - {key: 'messagesProcessed', title: 'event.min-messages-processed'}, - {key: 'errorsOccurred', title: 'event.min-errors-occurred'} + {key: 'minMessagesProcessed', title: 'event.min-messages-processed'}, + {key: 'maxMessagesProcessed', title: 'event.max-messages-processed'}, + {key: 'minErrorsOccurred', title: 'event.min-errors-occurred'}, + {key: 'maxErrorsOccurred', title: 'event.max-errors-occurred'} ); break; case DebugEventType.DEBUG_RULE_NODE: - case DebugEventType.DEBUG_RULE_CHAIN: this.filterColumns.push( {key: 'msgDirectionType', title: 'event.type'}, {key: 'entityId', title: 'event.entity-id'}, - {key: 'entityName', title: 'event.entity-type'}, + {key: 'entityType', title: 'event.entity-type'}, + {key: 'msgId', title: 'event.message-id'}, {key: 'msgType', title: 'event.message-type'}, {key: 'relationType', title: 'event.relation-type'}, {key: 'dataSearch', title: 'event.data'}, @@ -352,6 +376,13 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_RULE_CHAIN: + this.filterColumns.push( + {key: 'message', title: 'event.message'}, + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index 220e7ce3b2..89e93d4fa1 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -65,7 +65,7 @@ export interface StatsEventBody extends BaseEventBody { export interface DebugRuleNodeEventBody extends BaseEventBody { type: string; entityId: string; - entityName: string; + entityType: string; msgId: string; msgType: string; relationType: string; @@ -75,7 +75,12 @@ export interface DebugRuleNodeEventBody extends BaseEventBody { error: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody; +export interface DebugRuleChainEventBody extends BaseEventBody { + message: string; + error?: string; +} + +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; export interface Event extends BaseData { tenantId: TenantId; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index fd1bc2d688..6a7226860d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2012,6 +2012,7 @@ "body": "Body", "method": "Method", "type": "Type", + "message": "Message", "message-id": "Message Id", "message-type": "Message Type", "data-type": "Data Type", @@ -2023,8 +2024,10 @@ "success": "Success", "failed": "Failed", "messages-processed": "Messages processed", + "max-messages-processed": "Maximum messages processed", "min-messages-processed": "Minimum messages processed", "errors-occurred": "Errors occurred", + "max-errors-occurred": "Maximum errors occurred", "min-errors-occurred": "Minimum errors occurred", "min-value": "Minimum value is 0.", "all-events": "All",