From a720009b55678d30ab5f685ba3acc65934011b2d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 22 Jul 2022 18:53:41 +0300 Subject: [PATCH 01/13] Initial DB structure --- .../server/actors/ActorSystemContext.java | 159 +++++----- .../server/actors/stats/StatsActor.java | 18 +- .../server/controller/EventController.java | 8 +- .../controller/RuleChainController.java | 7 +- .../server/actors/stats/StatsActorTest.java | 3 +- .../AbstractRuleEngineControllerTest.java | 12 +- ...AbstractRuleEngineFlowIntegrationTest.java | 19 +- ...actRuleEngineLifecycleIntegrationTest.java | 18 +- .../server/dao/event/EventService.java | 13 +- .../data/{Event.java => EventInfo.java} | 8 +- .../server/common/data/event/ErrorEvent.java | 48 +++ .../server/common/data/event/Event.java | 42 +++ .../server/common/data/event/EventType.java | 12 +- .../common/data/event/LifecycleEvent.java | 51 ++++ .../data/event/RuleChainDebugEvent.java | 50 +++ .../common/data/event/RuleNodeDebugEvent.java | 75 +++++ .../common/data/event/StatisticsEvent.java | 47 +++ .../server/dao/event/BaseEventService.java | 59 +++- .../server/dao/event/EventDao.java | 13 +- .../server/dao/model/sql/EventEntity.java | 9 +- .../service/validator/EventDataValidator.java | 12 +- .../dao/sql/event/EventInsertRepository.java | 201 ++++++++++-- .../server/dao/sql/event/JpaBaseEventDao.java | 287 ++++++++++-------- .../insert/sql/SqlPartitioningRepository.java | 4 +- .../dao/sqlts/sql/JpaSqlTimeseriesDao.java | 2 +- .../server/dao/timeseries/SqlPartition.java | 10 +- .../main/resources/sql/schema-entities.sql | 68 ++++- .../dao/service/AbstractServiceTest.java | 20 +- .../service/event/BaseEventServiceTest.java | 50 +-- .../dao/sql/event/JpaBaseEventDaoTest.java | 249 ++++++++------- .../thingsboard/rest/client/RestClient.java | 10 +- 31 files changed, 1074 insertions(+), 510 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/{Event.java => EventInfo.java} (93%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index a38de7b839..e6f207d960 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -37,10 +37,11 @@ import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; import org.thingsboard.server.actors.service.ActorService; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.msg.TbActorMsg; @@ -112,6 +113,29 @@ import java.util.concurrent.TimeUnit; @Component public class ActorSystemContext { + private static final FutureCallback RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Rule Chain", th); + } + }; + private static final FutureCallback RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK = new FutureCallback<>() { + @Override + public void onSuccess(@Nullable Void event) { + + } + + @Override + public void onFailure(Throwable th) { + log.error("Could not save debug Event for Node", th); + } + }; + protected final ObjectMapper mapper = new ObjectMapper(); private final ConcurrentMap debugPerTenantLimits = new ConcurrentHashMap<>(); @@ -462,25 +486,28 @@ public class ActorSystemContext { } public void persistError(TenantId tenantId, EntityId entityId, String method, Exception e) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.ERROR); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), method, toString(e))); - persistEvent(event); + eventService.saveAsync(ErrorEvent.builder() + .tenantId(tenantId) + .entityId(entityId) + .serviceId(getServiceId()) + .method(method) + .error(toString(e)).build()); } public void persistLifecycleEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent lcEvent, Exception e) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.LC_EVENT); - event.setBody(toBodyJson(serviceInfoProvider.getServiceInfo().getServiceId(), lcEvent, Optional.ofNullable(e))); - persistEvent(event); - } + LifecycleEvent.LifecycleEventBuilder event = LifecycleEvent.builder() + .tenantId(tenantId) + .entityId(entityId) + .serviceId(getServiceId()) + .lcEventType(lcEvent.name()); + + if (e != null) { + event.success(false).error(toString(e)); + } else { + event.success(false); + } - private void persistEvent(Event event) { - eventService.saveAsync(event); + eventService.saveAsync(event.build()); } private String toString(Throwable e) { @@ -489,21 +516,6 @@ public class ActorSystemContext { return sw.toString(); } - private JsonNode toBodyJson(String serviceId, ComponentLifecycleEvent event, Optional e) { - ObjectNode node = mapper.createObjectNode().put("server", serviceId).put("event", event.name()); - if (e.isPresent()) { - node = node.put("success", false); - node = node.put("error", toString(e.get())); - } else { - node = node.put("success", true); - } - return node; - } - - private JsonNode toBodyJson(String serviceId, String method, String body) { - return mapper.createObjectNode().put("server", serviceId).put("method", method).put("error", body); - } - public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) { return partitionService.resolve(serviceType, tenantId, entityId); } @@ -539,44 +551,27 @@ public class ActorSystemContext { private void persistDebugAsync(TenantId tenantId, EntityId entityId, String type, TbMsg tbMsg, String relationType, Throwable error, String failureMessage) { if (checkLimits(tenantId, tbMsg, error)) { try { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.DEBUG_RULE_NODE); - - String metadata = mapper.writeValueAsString(tbMsg.getMetaData().getData()); - - ObjectNode node = mapper.createObjectNode() - .put("type", type) - .put("server", getServiceId()) - .put("entityId", tbMsg.getOriginator().getId().toString()) - .put("entityName", tbMsg.getOriginator().getEntityType().name()) - .put("msgId", tbMsg.getId().toString()) - .put("msgType", tbMsg.getType()) - .put("dataType", tbMsg.getDataType().name()) - .put("relationType", relationType) - .put("data", tbMsg.getData()) - .put("metadata", metadata); + RuleNodeDebugEvent.RuleNodeDebugEventBuilder event = RuleNodeDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId) + .serviceId(getServiceId()) + .eventType(type) + .eventEntity(tbMsg.getOriginator()) + .msgId(tbMsg.getId()) + .msgType(tbMsg.getType()) + .dataType(tbMsg.getDataType().name()) + .relationType(relationType) + .data(tbMsg.getData()) + .metadata(mapper.writeValueAsString(tbMsg.getMetaData().getData())); if (error != null) { - node = node.put("error", toString(error)); + event.error(toString(error)); } else if (failureMessage != null) { - node = node.put("error", failureMessage); + event.error(failureMessage); } - event.setBody(node); - ListenableFuture future = eventService.saveAsync(event); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void event) { - - } - - @Override - public void onFailure(Throwable th) { - log.error("Could not save debug Event for Node", th); - } - }, MoreExecutors.directExecutor()); + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, RULE_NODE_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } catch (IOException ex) { log.warn("Failed to persist rule node debug message", ex); } @@ -603,33 +598,17 @@ public class ActorSystemContext { } private void persistRuleChainDebugModeEvent(TenantId tenantId, EntityId entityId, Throwable error) { - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(DataConstants.DEBUG_RULE_CHAIN); - - ObjectNode node = mapper.createObjectNode() - //todo: what fields are needed here? - .put("server", getServiceId()) - .put("message", "Reached debug mode rate limit!"); - + RuleChainDebugEvent.RuleChainDebugEventBuilder event = RuleChainDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId) + .serviceId(getServiceId()) + .message("Reached debug mode rate limit!"); if (error != null) { - node = node.put("error", toString(error)); + event.error(toString(error)); } - event.setBody(node); - ListenableFuture future = eventService.saveAsync(event); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void event) { - - } - - @Override - public void onFailure(Throwable th) { - log.error("Could not save debug Event for Rule Chain", th); - } - }, MoreExecutors.directExecutor()); + ListenableFuture future = eventService.saveAsync(event.build()); + Futures.addCallback(future, RULE_CHAIN_DEBUG_EVENT_ERROR_CALLBACK, MoreExecutors.directExecutor()); } public static Exception toException(Throwable error) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index 8cdf5cec02..fa5dedf66f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -20,13 +20,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActor; -import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorId; import org.thingsboard.server.actors.TbStringActorId; import org.thingsboard.server.actors.service.ContextAwareActor; import org.thingsboard.server.actors.service.ContextBasedCreator; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.TbActorMsg; @@ -54,12 +54,14 @@ public class StatsActor extends ContextAwareActor { if (msg.isEmpty()) { return; } - Event event = new Event(); - event.setEntityId(msg.getEntityId()); - event.setTenantId(msg.getTenantId()); - event.setType(DataConstants.STATS); - event.setBody(toBodyJson(systemContext.getServiceInfoProvider().getServiceId(), msg.getMessagesProcessed(), msg.getErrorsOccurred())); - systemContext.getEventService().saveAsync(event); + systemContext.getEventService().saveAsync(StatisticsEvent.builder() + .tenantId(msg.getTenantId()) + .entityId(msg.getEntityId()) + .serviceId(systemContext.getServiceInfoProvider().getServiceId()) + .messagesProcessed(msg.getMessagesProcessed()) + .errorsOccurred(msg.getErrorsOccurred()) + .build() + ); } private JsonNode toBodyJson(String serviceId, long messagesProcessed, long errorsOccurred) { diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index ac56978536..dd6c05749f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -29,7 +29,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -110,7 +110,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}/{eventType}", method = RequestMethod.GET) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @@ -153,7 +153,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.GET) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @@ -198,7 +198,7 @@ public class EventController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/events/{entityType}/{entityId}", method = RequestMethod.POST) @ResponseBody - public PageData getEvents( + public PageData getEvents( @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable(ENTITY_TYPE) String strEntityType, @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index faa1711fd9..143844504a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -40,7 +40,7 @@ import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -70,7 +70,6 @@ import org.thingsboard.server.service.script.RuleNodeJsScriptEngine; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -353,10 +352,10 @@ public class RuleChainController extends BaseController { RuleNodeId ruleNodeId = new RuleNodeId(toUUID(strRuleNodeId)); checkRuleNode(ruleNodeId, Operation.READ); TenantId tenantId = getCurrentUser().getTenantId(); - List events = eventService.findLatestEvents(tenantId, ruleNodeId, DataConstants.DEBUG_RULE_NODE, 2); + List events = eventService.findLatestEvents(tenantId, ruleNodeId, DataConstants.DEBUG_RULE_NODE, 2); JsonNode result = null; if (events != null) { - for (Event event : events) { + for (EventInfo event : events) { JsonNode body = event.getBody(); if (body.has("type") && body.get("type").asText().equals("IN")) { result = body; diff --git a/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java b/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java index 0e928b7dd6..d4d9456a3f 100644 --- a/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java +++ b/application/src/test/java/org/thingsboard/server/actors/stats/StatsActorTest.java @@ -18,7 +18,8 @@ package org.thingsboard.server.actors.stats; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index ab902b82c6..7335c10d13 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -60,20 +60,20 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle return doGet("/api/ruleChain/metadata/" + ruleChainId.getId().toString(), RuleChainMetaData.class); } - protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { + protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { return getEvents(tenantId, entityId, DataConstants.DEBUG_RULE_NODE, limit); } - protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { + protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { TimePageLink pageLink = new TimePageLink(limit); return doGetTypedWithTimePageLink("/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&", - new TypeReference>() { + new TypeReference>() { }, pageLink, entityId.getEntityType(), entityId.getId(), eventType, tenantId.getId()); } - protected JsonNode getMetadata(Event outEvent) { + protected JsonNode getMetadata(EventInfo outEvent) { String metaDataStr = outEvent.getBody().get("metadata").asText(); try { return mapper.readTree(metaDataStr); @@ -82,7 +82,7 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle } } - protected Predicate filterByCustomEvent() { + protected Predicate filterByCustomEvent() { return event -> event.getBody().get("msgType").textValue().equals("CUSTOM"); } diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 9d96fe9d9c..24c94adba6 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -30,9 +30,10 @@ import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; @@ -177,15 +178,15 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule actorSystem.tell(qMsg); Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); - PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); - List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); + PageData eventsPage = getDebugEvents(savedTenant.getId(), ruleChain.getFirstRuleNodeId(), 1000); + List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); Assert.assertEquals(2, events.size()); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(ruleChain.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(ruleChain.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); @@ -299,16 +300,16 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Mockito.verify(tbMsgCallback, Mockito.timeout(10000)).onSuccess(); - PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); - List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); + PageData eventsPage = getDebugEvents(savedTenant.getId(), rootRuleChain.getFirstRuleNodeId(), 1000); + List events = eventsPage.getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); Assert.assertEquals(2, events.size()); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(rootRuleChain.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index 5d86e6a2a5..c840cf4c9a 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -20,18 +20,14 @@ import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.test.context.support.TestPropertySourceUtils; -import org.testcontainers.containers.GenericContainer; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.rule.RuleChain; @@ -109,11 +105,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac Assert.assertNotNull(ruleChainFinal.getFirstRuleNodeId()); //TODO find out why RULE_NODE update event did not appear all the time - List rcEvents = Awaitility.await("Rule Node started successfully") + List rcEvents = Awaitility.await("Rule Node started successfully") .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000) + List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000) .getData().stream().filter(e -> { var body = e.getBody(); return body.has("event") && body.get("event").asText().equals("STARTED") @@ -145,11 +141,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac log.warn("awaiting tbMsgCallback"); Mockito.verify(tbMsgCallback, Mockito.timeout(TimeUnit.SECONDS.toMillis(TIMEOUT))).onSuccess(); log.warn("awaiting events"); - List events = Awaitility.await("get debug by custom event") + List events = Awaitility.await("get debug by custom event") .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000) + List debugEvents = getDebugEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), 1000) .getData().stream().filter(filterByCustomEvent()).collect(Collectors.toList()); log.warn("filtered debug events [{}]", debugEvents.size()); debugEvents.forEach((e) -> log.warn("event: {}", e)); @@ -158,11 +154,11 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac x -> x.size() == 2); log.warn("asserting.."); - Event inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); + EventInfo inEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.IN)).findFirst().get(); Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), inEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), inEvent.getBody().get("entityId").asText()); - Event outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); + EventInfo outEvent = events.stream().filter(e -> e.getBody().get("type").asText().equals(DataConstants.OUT)).findFirst().get(); Assert.assertEquals(ruleChainFinal.getFirstRuleNodeId(), outEvent.getEntityId()); Assert.assertEquals(device.getId().getId().toString(), outEvent.getBody().get("entityId").asText()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 83624df8e3..0d157e755d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -16,7 +16,8 @@ package org.thingsboard.server.dao.event; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -30,15 +31,15 @@ public interface EventService { ListenableFuture saveAsync(Event event); - Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid); + Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid); - PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink); + PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink); - PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink); + PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink); - List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); + List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); - PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); + PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); void removeEvents(TenantId tenantId, EntityId entityId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java similarity index 93% rename from common/data/src/main/java/org/thingsboard/server/common/data/Event.java rename to common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java index 29ab33d075..f6c4ee69fb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Event.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EventInfo.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; */ @Data @ApiModel -public class Event extends BaseData { +public class EventInfo extends BaseData { @ApiModelProperty(position = 1, value = "JSON object with Tenant Id.", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private TenantId tenantId; @@ -41,15 +41,15 @@ public class Event extends BaseData { @ApiModelProperty(position = 5, value = "Event body.", dataType = "com.fasterxml.jackson.databind.JsonNode") private transient JsonNode body; - public Event() { + public EventInfo() { super(); } - public Event(EventId id) { + public EventInfo(EventId id) { super(id); } - public Event(Event event) { + public EventInfo(EventInfo event) { super(event); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java new file mode 100644 index 0000000000..8e06177ca6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class ErrorEvent extends Event { + + private static final long serialVersionUID = 960461434033192571L; + + @Builder + private ErrorEvent(TenantId tenantId, EntityId entityId, String serviceId, String method, String error) { + super(tenantId, entityId, serviceId); + this.method = method; + this.error = error; + } + + @Getter @Setter + private String method; + @Getter @Setter + private String error; + + @Override + public EventType getType() { + return EventType.ERROR; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java new file mode 100644 index 0000000000..9c013e1486 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EventId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@EqualsAndHashCode(callSuper = true) +public abstract class Event extends BaseData { + + private final TenantId tenantId; + private final EntityId entityId; + private final String serviceId; + + public Event(TenantId tenantId, EntityId entityId, String serviceId) { + super(); + this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID; + this.entityId = entityId; + this.serviceId = serviceId; + } + + public abstract EventType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 2645e7cce3..8d58158521 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -15,6 +15,16 @@ */ package org.thingsboard.server.common.data.event; +import lombok.Getter; + public enum EventType { - ERROR, LC_EVENT, STATS, DEBUG_RULE_NODE, DEBUG_RULE_CHAIN + ERROR("error_event"), LC_EVENT("lc_event"), STATS("stats_event"), DEBUG_RULE_NODE("rule_node_debug_event"), DEBUG_RULE_CHAIN("rule_chain_debug_event"); + + @Getter + private final String table; + + EventType(String table) { + this.table = table; + } + } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java new file mode 100644 index 0000000000..217955af96 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class LifecycleEvent extends Event { + + private static final long serialVersionUID = -3247420461850911549L; + + @Builder + private LifecycleEvent(TenantId tenantId, EntityId entityId, String serviceId, String lcEventType, boolean success, String error) { + super(tenantId, entityId, serviceId); + this.lcEventType = lcEventType; + this.success = success; + this.error = error; + } + + @Getter + private final String lcEventType; + @Getter + private final boolean success; + @Getter @Setter + private String error; + + @Override + public EventType getType() { + return EventType.LC_EVENT; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java new file mode 100644 index 0000000000..ac8c16c2e5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class RuleChainDebugEvent extends Event { + + private static final long serialVersionUID = -386392236201116767L; + + @Builder + private RuleChainDebugEvent(TenantId tenantId, EntityId entityId, String serviceId, String message, String error) { + super(tenantId, entityId, serviceId); + this.message = message; + this.error = error; + } + + @Getter @Setter + private String message; + @Getter @Setter + private String error; + + @Override + public EventType getType() { + return EventType.DEBUG_RULE_CHAIN; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java new file mode 100644 index 0000000000..f9280d2d6b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class RuleNodeDebugEvent extends Event { + + private static final long serialVersionUID = -6575797430064573984L; + + @Builder + private RuleNodeDebugEvent(TenantId tenantId, EntityId entityId, String serviceId, + String eventType, EntityId eventEntity, UUID msgId, + String msgType, String dataType, String relationType, + String data, String metadata, String error) { + super(tenantId, entityId, serviceId); + this.eventType = eventType; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.dataType = dataType; + this.relationType = relationType; + this.data = data; + this.metadata = metadata; + this.error = error; + } + + @Getter + private final String eventType; + @Getter + private final EntityId eventEntity; + @Getter + private final UUID msgId; + @Getter + private final String msgType; + @Getter + private final String dataType; + @Getter + private final String relationType; + @Getter @Setter + private String data; + @Getter @Setter + private String metadata; + @Getter @Setter + private String error; + + //TODO: rename the enum constant + @Override + public EventType getType() { + return EventType.DEBUG_RULE_NODE; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java new file mode 100644 index 0000000000..b0194762de --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@ToString +@EqualsAndHashCode(callSuper = true) +public class StatisticsEvent extends Event { + + private static final long serialVersionUID = 6683733979448910631L; + + @Builder + private StatisticsEvent(TenantId tenantId, EntityId entityId, String serviceId, long messagesProcessed, long errorsOccurred) { + super(tenantId, entityId, serviceId); + this.messagesProcessed = messagesProcessed; + this.errorsOccurred = errorsOccurred; + } + + @Getter + private final long messagesProcessed; + @Getter + private final long errorsOccurred; + + @Override + public EventType getType() { + return EventType.STATS; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 3937bbd9c3..d7f0885bdc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -15,15 +15,19 @@ */ package org.thingsboard.server.dao.event; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +38,8 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; +import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.stream.Collectors; @Service @@ -57,18 +63,41 @@ public class BaseEventService implements EventService { } private void checkAndTruncateDebugEvent(Event event) { - if (event.getType().startsWith("DEBUG") && event.getBody() != null && event.getBody().has("data")) { - String dataStr = event.getBody().get("data").asText(); - int length = dataStr.length(); + switch (event.getType()) { + case DEBUG_RULE_NODE: + RuleNodeDebugEvent rnEvent = (RuleNodeDebugEvent) event; + truncateField(rnEvent, RuleNodeDebugEvent::getData, RuleNodeDebugEvent::setData); + truncateField(rnEvent, RuleNodeDebugEvent::getMetadata, RuleNodeDebugEvent::setMetadata); + truncateField(rnEvent, RuleNodeDebugEvent::getError, RuleNodeDebugEvent::setError); + break; + case DEBUG_RULE_CHAIN: + RuleChainDebugEvent rcEvent = (RuleChainDebugEvent) event; + truncateField(rcEvent, RuleChainDebugEvent::getMessage, RuleChainDebugEvent::setMessage); + truncateField(rcEvent, RuleChainDebugEvent::getError, RuleChainDebugEvent::setError); + break; + case LC_EVENT: + LifecycleEvent lcEvent = (LifecycleEvent) event; + truncateField(lcEvent, LifecycleEvent::getError, LifecycleEvent::setError); + break; + case ERROR: + ErrorEvent eEvent = (ErrorEvent) event; + truncateField(eEvent, ErrorEvent::getError, ErrorEvent::setError); + break; + } + } + + private void truncateField(T event, Function getter, BiConsumer setter) { + var str = getter.apply(event); + if (StringUtils.isNotEmpty(str)) { + var length = str.length(); if (length > maxDebugEventSymbols) { - ((ObjectNode) event.getBody()).put("data", dataStr.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]"); - log.trace("[{}] Event was truncated: {}", event.getId(), dataStr); + setter.accept(event, str.substring(0, maxDebugEventSymbols) + "...[truncated " + (length - maxDebugEventSymbols) + " symbols]"); } } } @Override - public Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) { + public Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) { if (tenantId == null) { throw new DataValidationException("Tenant id should be specified!."); } @@ -81,27 +110,27 @@ public class BaseEventService implements EventService { if (StringUtils.isEmpty(eventUid)) { throw new DataValidationException("Event uid should be specified!."); } - Event event = eventDao.findEvent(tenantId.getId(), entityId, eventType, eventUid); + EventInfo event = eventDao.findEvent(tenantId.getId(), entityId, eventType, eventUid); return event != null ? Optional.of(event) : Optional.empty(); } @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink) { + public PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink) { return eventDao.findEvents(tenantId.getId(), entityId, pageLink); } @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { + public PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { return eventDao.findEvents(tenantId.getId(), entityId, eventType, pageLink); } @Override - public List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) { + public List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) { return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit); } @Override - public PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { + public PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { return eventDao.findEventByFilter(tenantId.getId(), entityId, eventFilter, pageLink); } @@ -113,7 +142,7 @@ public class BaseEventService implements EventService { @Override public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) { TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime); - PageData eventsPageData; + PageData eventsPageData; do { if (eventFilter == null) { eventsPageData = findEvents(tenantId, entityId, eventsPageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 86eb0d91ec..30815bd15e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -16,7 +16,8 @@ package org.thingsboard.server.dao.event; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageData; @@ -48,7 +49,7 @@ public interface EventDao extends Dao { * @param eventUid the eventUid * @return the event */ - Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid); + EventInfo findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid); /** * Find events by tenantId, entityId and pageLink. @@ -58,7 +59,7 @@ public interface EventDao extends Dao { * @param pageLink the pageLink * @return the event list */ - PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink); + PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink); /** * Find events by tenantId, entityId, eventType and pageLink. @@ -69,9 +70,9 @@ public interface EventDao extends Dao { * @param pageLink the pageLink * @return the event list */ - PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink); + PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink); - PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); + PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); /** * Find latest events by tenantId, entityId and eventType. @@ -82,7 +83,7 @@ public interface EventDao extends Dao { * @param limit the limit * @return the event list */ - List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit); + List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit); /** * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index b452ec5bda..bf08333e7b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -22,7 +22,8 @@ import lombok.NoArgsConstructor; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; @@ -78,7 +79,7 @@ public class EventEntity extends BaseSqlEntity implements BaseEntity implements BaseEntity { @Override protected void validateDataImpl(TenantId tenantId, Event event) { + if (event.getTenantId() == null) { + throw new DataValidationException("Tenant id should be specified!."); + } if (event.getEntityId() == null) { throw new DataValidationException("Entity id should be specified!."); } - if (StringUtils.isEmpty(event.getType())) { - throw new DataValidationException("Event type should be specified!."); - } - if (event.getBody() == null) { - throw new DataValidationException("Event body should be specified!."); + if (StringUtils.isEmpty(event.getServiceId())) { + throw new DataValidationException("Service id should be specified!."); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 6234db58da..e69b8578df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.event; +import org.jetbrains.annotations.NotNull; +import org.postgresql.core.Oid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.BatchPreparedStatementSetter; @@ -24,12 +26,24 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.server.dao.model.sql.EventEntity; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import javax.annotation.PostConstruct; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Types; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; @Repository @Transactional @@ -39,10 +53,7 @@ public class EventInsertRepository { private static final String EMPTY_STR = ""; - private static final String INSERT = - "INSERT INTO event (id, created_time, body, entity_id, entity_type, event_type, event_uid, tenant_id, ts) " + - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " + - "ON CONFLICT DO NOTHING;"; + private final Map insertStmtMap = new ConcurrentHashMap<>(); @Autowired protected JdbcTemplate jdbcTemplate; @@ -53,34 +64,172 @@ public class EventInsertRepository { @Value("${sql.remove_null_chars:true}") private boolean removeNullChars; - protected void save(List entities) { + @PostConstruct + public void init() { + insertStmtMap.put(EventType.ERROR, "INSERT INTO " + EventType.ERROR.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_method, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.LC_EVENT, "INSERT INTO " + EventType.LC_EVENT.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_type, e_success, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.STATS, "INSERT INTO " + EventType.STATS.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_messages_processed, e_errors_occured) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_RULE_NODE, "INSERT INTO " + EventType.DEBUG_RULE_NODE.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_type, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_data_type, e_relation_type, e_data, e_metadata, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + insertStmtMap.put(EventType.DEBUG_RULE_CHAIN, "INSERT INTO " + EventType.DEBUG_RULE_CHAIN.getTable() + + " (id, tenant_id, ts, entity_id, service_id, e_message, e_error) " + + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); + } + + protected void save(List entities) { + Map> eventsByType = entities.stream().collect(Collectors.groupingBy(Event::getType, Collectors.toList())); transactionTemplate.execute(new TransactionCallbackWithoutResult() { @Override protected void doInTransactionWithoutResult(TransactionStatus status) { - jdbcTemplate.batchUpdate(INSERT, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - EventEntity event = entities.get(i); - ps.setObject(1, event.getId()); - ps.setLong(2, event.getCreatedTime()); - ps.setString(3, replaceNullChars(event.getBody().toString())); - ps.setObject(4, event.getEntityId()); - ps.setString(5, event.getEntityType().name()); - ps.setString(6, event.getEventType()); - ps.setString(7, event.getEventUid()); - ps.setObject(8, event.getTenantId()); - ps.setLong(9, event.getTs()); - } - - @Override - public int getBatchSize() { - return entities.size(); - } - }); + for (var entry : eventsByType.entrySet()) { + jdbcTemplate.batchUpdate(insertStmtMap.get(entry.getKey()), getStatementSetter(entry.getKey(), entry.getValue())); + } } }); } + private BatchPreparedStatementSetter getStatementSetter(EventType eventType, List events) { + switch (eventType) { + case ERROR: + return getErrorEventSetter(events); + case LC_EVENT: + return getLcEventSetter(events); + case STATS: + return getStatsEventSetter(events); + case DEBUG_RULE_NODE: + return getRuleNodeEventSetter(events); + case DEBUG_RULE_CHAIN: + return getRuleChainEventSetter(events); + default: + throw new RuntimeException(eventType + " support is not implemented!"); + } + } + + private BatchPreparedStatementSetter getErrorEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ErrorEvent event = (ErrorEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getMethod()); + safePutString(ps, 7, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getLcEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + LifecycleEvent event = (LifecycleEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getLcEventType()); + ps.setBoolean(7, event.isSuccess()); + safePutString(ps, 8, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getStatsEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + StatisticsEvent event = (StatisticsEvent) events.get(i); + setCommonEventFields(ps, event); + ps.setLong(6, event.getMessagesProcessed()); + ps.setLong(7, event.getErrorsOccurred()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getRuleNodeEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RuleNodeDebugEvent event = (RuleNodeDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getEventType()); + safePutUUID(ps, 7, event.getEntityId() != null ? event.getEntityId().getId() : null); + safePutString(ps, 8, event.getEntityId() != null ? event.getEntityId().getEntityType().name() : null); + safePutUUID(ps, 9, event.getMsgId()); + safePutString(ps, 10, event.getMsgType()); + safePutString(ps, 11, event.getDataType()); + safePutString(ps, 12, event.getRelationType()); + safePutString(ps, 13, event.getData()); + safePutString(ps, 14, event.getMetadata()); + safePutString(ps, 15, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + private BatchPreparedStatementSetter getRuleChainEventSetter(List events) { + return new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RuleChainDebugEvent event = (RuleChainDebugEvent) events.get(i); + setCommonEventFields(ps, event); + safePutString(ps, 6, event.getMessage()); + safePutString(ps, 7, event.getError()); + } + + @Override + public int getBatchSize() { + return events.size(); + } + }; + } + + void safePutString(PreparedStatement ps, int parameterIdx, String value) throws SQLException { + if (value != null) { + ps.setString(parameterIdx, replaceNullChars(value)); + } else { + ps.setNull(parameterIdx, Types.VARCHAR); + } + } + + void safePutUUID(PreparedStatement ps, int parameterIdx, UUID value) throws SQLException { + if (value != null) { + ps.setObject(parameterIdx, value); + } else { + ps.setNull(parameterIdx, Types.OTHER); + } + } + + private void setCommonEventFields(PreparedStatement ps, Event event) throws SQLException { + ps.setObject(1, event.getId().getId()); + ps.setObject(2, event.getTenantId().getId()); + ps.setLong(3, event.getCreatedTime()); + ps.setObject(4, event.getEntityId().getId()); + ps.setString(5, event.getServiceId()); + } + private String replaceNullChars(String strValue) { if (removeNullChars && strValue != null) { return PATTERN_THREAD_LOCAL.get().matcher(strValue).replaceAll(EMPTY_STR); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 2b409006d2..deb3c87402 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -19,15 +19,19 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.DebugEvent; import org.thingsboard.server.common.data.event.ErrorEventFilter; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifeCycleEventFilter; import org.thingsboard.server.common.data.event.StatisticsEventFilter; import org.thingsboard.server.common.data.id.EntityId; @@ -42,17 +46,27 @@ import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.timeseries.SqlPartition; +import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; - /** * Created by Valerii Sosliuk on 5/3/2017. */ @@ -60,7 +74,12 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Component public class JpaBaseEventDao extends JpaAbstractDao implements EventDao { - private final UUID systemTenantId = NULL_UUID; + private static final long PARTITION_DURATION = TimeUnit.HOURS.toMillis(1); + private final Map> partitionsByEventType = new ConcurrentHashMap<>(); + private static final ReentrantLock partitionCreationLock = new ReentrantLock(); + + @Autowired + private SqlPartitioningRepository partitioningRepository; @Autowired private EventRepository eventRepository; @@ -102,10 +121,13 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen @Value("${sql.batch_sort:false}") private boolean batchSortEnabled; - private TbSqlBlockingQueueWrapper queue; + private TbSqlBlockingQueueWrapper queue; @PostConstruct private void init() { + for (EventType eventType : EventType.values()) { + partitionsByEventType.put(eventType, new ConcurrentHashMap<>()); + } TbSqlBlockingQueueParams params = TbSqlBlockingQueueParams.builder() .logName("Events") .batchSize(batchSize) @@ -114,11 +136,9 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen .statsNamePrefix("events") .batchSortEnabled(batchSortEnabled) .build(); - Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); + Function hashcodeFunction = entity -> Objects.hash(super.hashCode(), entity.getTenantId(), entity.getEntityId()); queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory); - queue.init(logExecutor, v -> eventInsertRepository.save(v), - Comparator.comparing((EventEntity eventEntity) -> eventEntity.getTs()) - ); + queue.init(logExecutor, v -> eventInsertRepository.save(v), Comparator.comparing(Event::getCreatedTime)); } @PreDestroy @@ -143,68 +163,80 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen event.setCreatedTime(System.currentTimeMillis()); } } - if (StringUtils.isEmpty(event.getUid())) { - event.setUid(event.getId().toString()); - } - - return save(new EventEntity(event)); + savePartitionIfNotExist(event); + return queue.add(event); } - private ListenableFuture save(EventEntity entity) { - log.debug("Save event [{}] ", entity); - if (entity.getTenantId() == null) { - log.trace("Save system event with predefined id {}", systemTenantId); - entity.setTenantId(systemTenantId); - } - if (entity.getUuid() == null) { - entity.setUuid(Uuids.timeBased()); - } - if (StringUtils.isEmpty(entity.getEventUid())) { - entity.setEventUid(entity.getUuid().toString()); + private void savePartitionIfNotExist(Event event) { + var partitionsMap = partitionsByEventType.get(event.getType()); + long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % PARTITION_DURATION); + if (partitionsMap.get(partitionStartTs) == null) { + long partitionEndTs = partitionStartTs + PARTITION_DURATION; + savePartition(partitionsMap, new SqlPartition(event.getType().getTable(), partitionStartTs, partitionEndTs, Long.toString(partitionStartTs))); } - return addToQueue(entity); } - private ListenableFuture addToQueue(EventEntity entity) { - return queue.add(entity); + private void savePartition(Map partitionsMap, SqlPartition sqlPartition) { + if (!partitionsMap.containsKey(sqlPartition.getStart())) { + partitionCreationLock.lock(); + try { + log.trace("Saving partition: {}", sqlPartition); + partitioningRepository.save(sqlPartition); + log.trace("Adding partition to map: {}", sqlPartition); + partitionsMap.put(sqlPartition.getStart(), sqlPartition); + } catch (DataIntegrityViolationException ex) { + log.trace("Error occurred during partition save:", ex); + if (ex.getCause() instanceof ConstraintViolationException) { + log.warn("Saving partition [{}] rejected. Event data will save to the DEFAULT partition.", sqlPartition.getPartitionDate()); + partitionsMap.put(sqlPartition.getStart(), sqlPartition); + } else { + throw new RuntimeException(ex); + } + } finally { + partitionCreationLock.unlock(); + } + } } @Override - public Event findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid) { - return DaoUtil.getData(eventRepository.findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid( - tenantId, entityId.getEntityType(), entityId.getId(), eventType, eventUid)); + public EventInfo findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid) { + return null; +// return DaoUtil.getData(eventRepository.findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid( +// tenantId, entityId.getEntityType(), entityId.getId(), eventType, eventUid)); } @Override - public PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository - .findEventsByTenantIdAndEntityId( - tenantId, - entityId.getEntityType(), - entityId.getId(), - Objects.toString(pageLink.getTextSearch(), ""), - pageLink.getStartTime(), - pageLink.getEndTime(), - DaoUtil.toPageable(pageLink))); + public PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) { + return null; +// return DaoUtil.toPageData( +// eventRepository +// .findEventsByTenantIdAndEntityId( +// tenantId, +// entityId.getEntityType(), +// entityId.getId(), +// Objects.toString(pageLink.getTextSearch(), ""), +// pageLink.getStartTime(), +// pageLink.getEndTime(), +// DaoUtil.toPageable(pageLink))); } @Override - public PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository - .findEventsByTenantIdAndEntityIdAndEventType( - tenantId, - entityId.getEntityType(), - entityId.getId(), - eventType, - pageLink.getStartTime(), - pageLink.getEndTime(), - DaoUtil.toPageable(pageLink))); + public PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { + return null; +// return DaoUtil.toPageData( +// eventRepository +// .findEventsByTenantIdAndEntityIdAndEventType( +// tenantId, +// entityId.getEntityType(), +// entityId.getId(), +// eventType, +// pageLink.getStartTime(), +// pageLink.getEndTime(), +// DaoUtil.toPageable(pageLink))); } @Override - public PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { + public PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { if (eventFilter.hasFilterForJsonBody()) { switch (eventFilter.getEventType()) { case DEBUG_RULE_NODE: @@ -224,86 +256,91 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen } } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, DebugEvent eventFilter, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository.findDebugRuleNodeEvents( - tenantId, - entityId.getId(), - entityId.getEntityType().name(), - eventFilter.getEventType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), - eventFilter.getMsgDirectionType(), - eventFilter.getServer(), - eventFilter.getEntityName(), - eventFilter.getRelationType(), - eventFilter.getEntityId(), - eventFilter.getMsgType(), - eventFilter.isError(), - eventFilter.getErrorStr(), - eventFilter.getDataSearch(), - eventFilter.getMetadataSearch(), - DaoUtil.toPageable(pageLink))); + private PageData findEventByFilter(UUID tenantId, EntityId entityId, DebugEvent eventFilter, TimePageLink pageLink) { + return null; +// return DaoUtil.toPageData( +// eventRepository.findDebugRuleNodeEvents( +// tenantId, +// entityId.getId(), +// entityId.getEntityType().name(), +// eventFilter.getEventType().name(), +// notNull(pageLink.getStartTime()), +// notNull(pageLink.getEndTime()), +// eventFilter.getMsgDirectionType(), +// eventFilter.getServer(), +// eventFilter.getEntityName(), +// eventFilter.getRelationType(), +// eventFilter.getEntityId(), +// eventFilter.getMsgType(), +// eventFilter.isError(), +// eventFilter.getErrorStr(), +// eventFilter.getDataSearch(), +// eventFilter.getMetadataSearch(), +// DaoUtil.toPageable(pageLink))); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository.findErrorEvents( - tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), - eventFilter.getServer(), - eventFilter.getMethod(), - eventFilter.getErrorStr(), - DaoUtil.toPageable(pageLink)) - ); + private PageData findEventByFilter(UUID tenantId, EntityId entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { + return null; +// return DaoUtil.toPageData( +// eventRepository.findErrorEvents( +// tenantId, +// entityId.getId(), +// entityId.getEntityType().name(), +// notNull(pageLink.getStartTime()), +// notNull(pageLink.getEndTime()), +// eventFilter.getServer(), +// eventFilter.getMethod(), +// eventFilter.getErrorStr(), +// DaoUtil.toPageable(pageLink)) +// ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { - boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); - boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); - return DaoUtil.toPageData( - eventRepository.findLifeCycleEvents( - tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), - eventFilter.getServer(), - eventFilter.getEvent(), - statusFilterEnabled, - statusFilter, - eventFilter.getErrorStr(), - DaoUtil.toPageable(pageLink)) - ); + private PageData findEventByFilter(UUID tenantId, EntityId entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { + return null; +// boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); +// boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); +// return DaoUtil.toPageData( +// eventRepository.findLifeCycleEvents( +// tenantId, +// entityId.getId(), +// entityId.getEntityType().name(), +// notNull(pageLink.getStartTime()), +// notNull(pageLink.getEndTime()), +// eventFilter.getServer(), +// eventFilter.getEvent(), +// statusFilterEnabled, +// statusFilter, +// eventFilter.getErrorStr(), +// DaoUtil.toPageable(pageLink)) +// ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { - return DaoUtil.toPageData( - eventRepository.findStatisticsEvents( - tenantId, - entityId.getId(), - entityId.getEntityType().name(), - notNull(pageLink.getStartTime()), - notNull(pageLink.getEndTime()), - eventFilter.getServer(), - notNull(eventFilter.getMessagesProcessed()), - notNull(eventFilter.getErrorsOccurred()), - DaoUtil.toPageable(pageLink)) - ); + private PageData findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { + return null; +// return DaoUtil.toPageData( +// eventRepository.findStatisticsEvents( +// tenantId, +// entityId.getId(), +// entityId.getEntityType().name(), +// notNull(pageLink.getStartTime()), +// notNull(pageLink.getEndTime()), +// eventFilter.getServer(), +// notNull(eventFilter.getMessagesProcessed()), +// notNull(eventFilter.getErrorsOccurred()), +// DaoUtil.toPageable(pageLink)) +// ); } @Override - public List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit) { - List latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( - tenantId, - entityId.getEntityType(), - entityId.getId(), - eventType, - PageRequest.of(0, limit)); - return DaoUtil.convertDataList(latest); + public List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit) { + return null; +// List latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( +// tenantId, +// entityId.getEntityType(), +// entityId.getId(), +// eventType, +// PageRequest.of(0, limit)); +// return DaoUtil.convertDataList(latest); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java index 899a195538..805e5e4f4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/sql/SqlPartitioningRepository.java @@ -23,7 +23,6 @@ import org.thingsboard.server.dao.util.SqlTsDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -@SqlTsDao @Repository @Transactional public class SqlPartitioningRepository { @@ -32,8 +31,7 @@ public class SqlPartitioningRepository { private EntityManager entityManager; public void save(SqlPartition partition) { - entityManager.createNativeQuery(partition.getQuery()) - .executeUpdate(); + entityManager.createNativeQuery(partition.getQuery()).executeUpdate(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java index ce173e046b..64d1669eae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java @@ -132,7 +132,7 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao long partitionEndTs = toMills(localDateTimeEnd); ZonedDateTime zonedDateTime = localDateTimeStart.atZone(ZoneOffset.UTC); String partitionDate = zonedDateTime.format(DateTimeFormatter.ofPattern(tsFormat.getPattern())); - savePartition(new SqlPartition(partitionStartTs, partitionEndTs, partitionDate)); + savePartition(new SqlPartition(SqlPartition.TS_KV, partitionStartTs, partitionEndTs, partitionDate)); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java index 5c47689b6d..dcdb335784 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/SqlPartition.java @@ -20,21 +20,21 @@ import lombok.Data; @Data public class SqlPartition { - private static final String TABLE_REGEX = "ts_kv_"; + public static final String TS_KV = "ts_kv"; private long start; private long end; private String partitionDate; private String query; - public SqlPartition(long start, long end, String partitionDate) { + public SqlPartition(String table, long start, long end, String partitionDate) { this.start = start; this.end = end; this.partitionDate = partitionDate; - this.query = createStatement(start, end, partitionDate); + this.query = createStatement(table, start, end, partitionDate); } - private String createStatement(long start, long end, String partitionDate) { - return "CREATE TABLE IF NOT EXISTS " + TABLE_REGEX + partitionDate + " PARTITION OF ts_kv FOR VALUES FROM (" + start + ") TO (" + end + ")"; + private String createStatement(String table, long start, long end, String partitionDate) { + return "CREATE TABLE IF NOT EXISTS " + table + "_" + partitionDate + " PARTITION OF " + table + " FOR VALUES FROM (" + start + ") TO (" + end + ")"; } } \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index a9b25a66f7..9494e4bdd6 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -323,18 +323,64 @@ CREATE TABLE IF NOT EXISTS device_credentials ( CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id) ); -CREATE TABLE IF NOT EXISTS event ( - id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY, - created_time bigint NOT NULL, - body varchar(10000000), - entity_id uuid, - entity_type varchar(255), - event_type varchar(255), - event_uid varchar(255), - tenant_id uuid, +CREATE TABLE IF NOT EXISTS rule_node_debug_event ( + id uuid NOT NULL CONSTRAINT rule_node_debug_event_pkey PRIMARY KEY, + tenant_id uuid NOT NULL , ts bigint NOT NULL, - CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) -); + entity_id uuid NOT NULL, + service_id varchar, + e_type varchar, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_data_type varchar, + e_relation_type varchar, + e_data varchar NOT NULL, + e_metadata varchar NOT NULL, + e_error varchar NOT NULL +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( + id uuid NOT NULL CONSTRAINT rule_chain_debug_event_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_message varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS stats_event ( + id uuid NOT NULL CONSTRAINT stats_event_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_messages_processed bigint NOT NULL, + e_errors_occured bigint NOT NULL +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS lc_event ( + id uuid NOT NULL CONSTRAINT lc_event_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_type varchar NOT NULL, + e_success boolean NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS error_event ( + id uuid NOT NULL CONSTRAINT error_event_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_method varchar NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS relation ( from_id uuid, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 964e6229f1..1276f46621 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -28,17 +28,20 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -185,17 +188,16 @@ public abstract class AbstractServiceTest { } - protected Event generateEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) throws IOException { + protected RuleNodeDebugEvent generateEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) throws IOException { if (tenantId == null) { tenantId = TenantId.fromUUID(Uuids.timeBased()); } - Event event = new Event(); - event.setTenantId(tenantId); - event.setEntityId(entityId); - event.setType(eventType); - event.setUid(eventUid); - event.setBody(readFromResource("TestJsonData.json")); - return event; + return RuleNodeDebugEvent.builder() + .tenantId(tenantId) + .entityId(entityId) + .serviceId("server A") + .data(JacksonUtil.toString(readFromResource("TestJsonData.json"))) + .build(); } // // private ComponentDescriptor getOrCreateDescriptor(ComponentScope scope, ComponentType type, String clazz, String configurationDescriptorResource) throws IOException { diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index f3f6dd8ff8..ffd15ff5ba 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -16,12 +16,13 @@ package org.thingsboard.server.dao.service.event; import com.datastax.oss.driver.api.core.uuid.Uuids; -import org.apache.commons.lang3.time.DateFormatUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -33,9 +34,6 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.service.AbstractServiceTest; import java.text.ParseException; -import java.time.LocalDateTime; -import java.time.Month; -import java.time.ZoneOffset; import java.util.Optional; import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; @@ -59,14 +57,15 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { @Test public void saveEvent() throws Exception { DeviceId devId = new DeviceId(Uuids.timeBased()); - Event event = generateEvent(null, devId, "ALARM", Uuids.timeBased().toString()); + RuleNodeDebugEvent event = generateEvent(null, devId, "ALARM", Uuids.timeBased().toString()); eventService.saveAsync(event).get(); - Optional loaded = eventService.findEvent(event.getTenantId(), event.getEntityId(), event.getType(), event.getUid()); - Assert.assertTrue(loaded.isPresent()); - Assert.assertNotNull(loaded.get()); - Assert.assertEquals(event.getEntityId(), loaded.get().getEntityId()); - Assert.assertEquals(event.getType(), loaded.get().getType()); - Assert.assertEquals(event.getBody(), loaded.get().getBody()); + throw new RuntimeException("fix me!"); +// Optional loaded = eventService.findEvent(event.getTenantId(), event.getEntityId(), event.getType(), event.getUid()); +// Assert.assertTrue(loaded.isPresent()); +// Assert.assertNotNull(loaded.get()); +// Assert.assertEquals(event.getEntityId(), loaded.get().getEntityId()); +// Assert.assertEquals(event.getType(), loaded.get().getType()); +// Assert.assertEquals(event.getBody(), loaded.get().getBody()); } @Test @@ -74,14 +73,14 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); - Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); - Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); + EventInfo savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); + EventInfo savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + EventInfo savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, + PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink); Assert.assertNotNull(events.getData()); @@ -105,14 +104,14 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); - Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); - Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); + EventInfo savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); + EventInfo savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + EventInfo savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, + PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink); Assert.assertNotNull(events.getData()); @@ -131,10 +130,11 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); } - private Event saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { - Event event = generateEvent(tenantId, entityId, DataConstants.STATS, null); - event.setId(new EventId(Uuids.startOf(time))); - eventService.saveAsync(event).get(); - return event; + private EventInfo saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { + throw new RuntimeException("fix me!"); +// EventInfo event = generateEvent(tenantId, entityId, DataConstants.STATS, null); +// event.setId(new EventId(Uuids.startOf(time))); +// eventService.saveAsync(event).get(); +// return event; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java index eb5f004714..4381aab18f 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java @@ -23,7 +23,7 @@ import org.junit.After; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; @@ -35,9 +35,7 @@ import org.thingsboard.server.dao.event.EventDao; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.UUID; -import java.util.concurrent.ExecutionException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -58,128 +56,129 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { @After public void deleteEvents() { - List events = eventDao.find(TenantId.fromUUID(tenantId)); - for (Event event : events) { - eventDao.removeById(TenantId.fromUUID(tenantId), event.getUuidId()); - } + throw new RuntimeException("fix me!"); +// List events = eventDao.find(TenantId.fromUUID(tenantId)); +// for (EventInfo event : events) { +// eventDao.removeById(TenantId.fromUUID(tenantId), event.getUuidId()); +// } } - @Test - public void findEvent() { - UUID entityId = Uuids.timeBased(); - Event savedEvent = eventDao.save(TenantId.fromUUID(tenantId), getEvent(entityId, tenantId, entityId)); - Event foundEvent = eventDao.findEvent(tenantId, new DeviceId(entityId), DataConstants.STATS, savedEvent.getUid()); - assertNotNull("Event expected to be not null", foundEvent); - assertEquals(savedEvent.getId(), foundEvent.getId()); - } - - @Test - public void findEventsByEntityIdAndPageLink() throws Exception { - UUID entityId1 = Uuids.timeBased(); - UUID entityId2 = Uuids.timeBased(); - long startTime = System.currentTimeMillis(); - long endTime = createEventsTwoEntities(tenantId, entityId1, entityId2, 20); - - TimePageLink pageLink1 = new TimePageLink(30); - PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink1); - assertEquals(10, events1.getData().size()); - - TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); - PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink2); - assertEquals(10, events2.getData().size()); - - TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); - PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink3); - assertEquals(10, events3.getData().size()); - - TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); - PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(5, events4.getData().size()); - - pageLink4 = pageLink4.nextPageLink(); - PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(5, events5.getData().size()); - - pageLink4 = pageLink4.nextPageLink(); - PageData events6 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); - assertEquals(0, events6.getData().size()); - - } - - @Test - public void findEventsByEntityIdAndEventTypeAndPageLink() throws Exception { - UUID entityId1 = Uuids.timeBased(); - UUID entityId2 = Uuids.timeBased(); - long startTime = System.currentTimeMillis(); - long endTime = createEventsTwoEntitiesTwoTypes(tenantId, entityId1, entityId2, 20); - - TimePageLink pageLink1 = new TimePageLink(30); - PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink1); - assertEquals(5, events1.getData().size()); - - TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); - PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink2); - assertEquals(5, events2.getData().size()); - - TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); - PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink3); - assertEquals(5, events3.getData().size()); - - TimePageLink pageLink4 = new TimePageLink(4, 0, "", null, startTime, endTime); - PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); - assertEquals(4, events4.getData().size()); - - pageLink4 = pageLink4.nextPageLink(); - PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); - assertEquals(1, events5.getData().size()); - } - - private long createEventsTwoEntitiesTwoTypes(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { - for (int i = 0; i < count / 2; i++) { - String type = i % 2 == 0 ? STATS : ALARM; - UUID eventId1 = Uuids.timeBased(); - Event event1 = getEvent(eventId1, tenantId, entityId1, type); - eventDao.saveAsync(event1).get(); - UUID eventId2 = Uuids.timeBased(); - Event event2 = getEvent(eventId2, tenantId, entityId2, type); - eventDao.saveAsync(event2).get(); - } - return System.currentTimeMillis(); - } - - private long createEventsTwoEntities(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { - for (int i = 0; i < count / 2; i++) { - UUID eventId1 = Uuids.timeBased(); - Event event1 = getEvent(eventId1, tenantId, entityId1); - eventDao.saveAsync(event1).get(); - UUID eventId2 = Uuids.timeBased(); - Event event2 = getEvent(eventId2, tenantId, entityId2); - eventDao.saveAsync(event2).get(); - } - return System.currentTimeMillis(); - } - - private Event getEvent(UUID eventId, UUID tenantId, UUID entityId, String type) { - Event event = getEvent(eventId, tenantId, entityId); - event.setType(type); - return event; - } - - private Event getEvent(UUID eventId, UUID tenantId, UUID entityId) { - Event event = new Event(); - event.setId(new EventId(eventId)); - event.setTenantId(TenantId.fromUUID(tenantId)); - EntityId deviceId = new DeviceId(entityId); - event.setEntityId(deviceId); - event.setUid(event.getId().getId().toString()); - event.setType(STATS); - ObjectMapper mapper = new ObjectMapper(); - try { - JsonNode jsonNode = mapper.readTree("{\"key\":\"value\"}"); - event.setBody(jsonNode); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - return event; - } +// @Test +// public void findEvent() { +// UUID entityId = Uuids.timeBased(); +// EventInfo savedEvent = eventDao.save(TenantId.fromUUID(tenantId), getEvent(entityId, tenantId, entityId)); +// EventInfo foundEvent = eventDao.findEvent(tenantId, new DeviceId(entityId), DataConstants.STATS, savedEvent.getUid()); +// assertNotNull("Event expected to be not null", foundEvent); +// assertEquals(savedEvent.getId(), foundEvent.getId()); +// } +// +// @Test +// public void findEventsByEntityIdAndPageLink() throws Exception { +// UUID entityId1 = Uuids.timeBased(); +// UUID entityId2 = Uuids.timeBased(); +// long startTime = System.currentTimeMillis(); +// long endTime = createEventsTwoEntities(tenantId, entityId1, entityId2, 20); +// +// TimePageLink pageLink1 = new TimePageLink(30); +// PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink1); +// assertEquals(10, events1.getData().size()); +// +// TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); +// PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink2); +// assertEquals(10, events2.getData().size()); +// +// TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); +// PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink3); +// assertEquals(10, events3.getData().size()); +// +// TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); +// PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); +// assertEquals(5, events4.getData().size()); +// +// pageLink4 = pageLink4.nextPageLink(); +// PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); +// assertEquals(5, events5.getData().size()); +// +// pageLink4 = pageLink4.nextPageLink(); +// PageData events6 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); +// assertEquals(0, events6.getData().size()); +// +// } +// +// @Test +// public void findEventsByEntityIdAndEventTypeAndPageLink() throws Exception { +// UUID entityId1 = Uuids.timeBased(); +// UUID entityId2 = Uuids.timeBased(); +// long startTime = System.currentTimeMillis(); +// long endTime = createEventsTwoEntitiesTwoTypes(tenantId, entityId1, entityId2, 20); +// +// TimePageLink pageLink1 = new TimePageLink(30); +// PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink1); +// assertEquals(5, events1.getData().size()); +// +// TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); +// PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink2); +// assertEquals(5, events2.getData().size()); +// +// TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); +// PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink3); +// assertEquals(5, events3.getData().size()); +// +// TimePageLink pageLink4 = new TimePageLink(4, 0, "", null, startTime, endTime); +// PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); +// assertEquals(4, events4.getData().size()); +// +// pageLink4 = pageLink4.nextPageLink(); +// PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); +// assertEquals(1, events5.getData().size()); +// } +// +// private long createEventsTwoEntitiesTwoTypes(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { +// for (int i = 0; i < count / 2; i++) { +// String type = i % 2 == 0 ? STATS : ALARM; +// UUID eventId1 = Uuids.timeBased(); +// EventInfo event1 = getEvent(eventId1, tenantId, entityId1, type); +// eventDao.saveAsync(event1).get(); +// UUID eventId2 = Uuids.timeBased(); +// EventInfo event2 = getEvent(eventId2, tenantId, entityId2, type); +// eventDao.saveAsync(event2).get(); +// } +// return System.currentTimeMillis(); +// } +// +// private long createEventsTwoEntities(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { +// for (int i = 0; i < count / 2; i++) { +// UUID eventId1 = Uuids.timeBased(); +// EventInfo event1 = getEvent(eventId1, tenantId, entityId1); +// eventDao.saveAsync(event1).get(); +// UUID eventId2 = Uuids.timeBased(); +// EventInfo event2 = getEvent(eventId2, tenantId, entityId2); +// eventDao.saveAsync(event2).get(); +// } +// return System.currentTimeMillis(); +// } +// +// private EventInfo getEvent(UUID eventId, UUID tenantId, UUID entityId, String type) { +// EventInfo event = getEvent(eventId, tenantId, entityId); +// event.setType(type); +// return event; +// } +// +// private EventInfo getEvent(UUID eventId, UUID tenantId, UUID entityId) { +// EventInfo event = new EventInfo(); +// event.setId(new EventId(eventId)); +// event.setTenantId(TenantId.fromUUID(tenantId)); +// EntityId deviceId = new DeviceId(entityId); +// event.setEntityId(deviceId); +// event.setUid(event.getId().getId().toString()); +// event.setType(STATS); +// ObjectMapper mapper = new ObjectMapper(); +// try { +// JsonNode jsonNode = mapper.readTree("{\"key\":\"value\"}"); +// event.setBody(jsonNode); +// } catch (IOException e) { +// log.error(e.getMessage(), e); +// } +// return event; +// } } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index a0f1d5dfa1..f2f446e534 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -54,7 +54,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.Event; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveDeviceWithCredentialsRequest; @@ -1811,7 +1811,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } } - public PageData getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) { + public PageData getEvents(EntityId entityId, String eventType, TenantId tenantId, TimePageLink pageLink) { Map params = new HashMap<>(); params.put("entityType", entityId.getEntityType().name()); params.put("entityId", entityId.getId().toString()); @@ -1823,12 +1823,12 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { baseURL + "/api/events/{entityType}/{entityId}/{eventType}?tenantId={tenantId}&" + getTimeUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, - new ParameterizedTypeReference>() { + new ParameterizedTypeReference>() { }, params).getBody(); } - public PageData getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) { + public PageData getEvents(EntityId entityId, TenantId tenantId, TimePageLink pageLink) { Map params = new HashMap<>(); params.put("entityType", entityId.getEntityType().name()); params.put("entityId", entityId.getId().toString()); @@ -1839,7 +1839,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { baseURL + "/api/events/{entityType}/{entityId}?tenantId={tenantId}&" + getTimeUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, - new ParameterizedTypeReference>() { + new ParameterizedTypeReference>() { }, params).getBody(); } From e89554aaf421112cddc317793021b91d7435197c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 25 Jul 2022 19:36:13 +0300 Subject: [PATCH 02/13] WIP: improvement to fetch of the events --- .../server/actors/ActorSystemContext.java | 10 +- .../server/actors/stats/StatsActor.java | 2 +- .../controller/ControllerConstants.java | 2 +- .../server/controller/EventController.java | 62 +-- .../controller/RuleChainController.java | 3 +- .../AbstractRuleEngineControllerTest.java | 4 +- ...actRuleEngineLifecycleIntegrationTest.java | 3 +- .../server/dao/event/EventService.java | 9 +- .../server/common/data/DataConstants.java | 6 - .../server/common/data/event/ErrorEvent.java | 13 +- .../server/common/data/event/Event.java | 31 +- .../server/common/data/event/EventType.java | 7 +- .../common/data/event/LifecycleEvent.java | 29 +- .../data/event/RuleChainDebugEvent.java | 11 +- .../common/data/event/RuleNodeDebugEvent.java | 11 +- .../common/data/event/StatisticsEvent.java | 13 +- .../server/dao/event/BaseEventService.java | 70 ++-- .../server/dao/event/EventDao.java | 20 +- .../server/dao/model/ModelConstants.java | 21 +- .../dao/model/sql/ErrorEventEntity.java | 73 ++++ .../server/dao/model/sql/EventEntity.java | 114 ++---- .../dao/model/sql/LifecycleEventEntity.java | 72 ++++ .../dao/model/sql/StatisticsEventEntity.java | 72 ++++ .../dao/sql/event/ErrorEventRepository.java | 74 ++++ .../dao/sql/event/EventInsertRepository.java | 8 +- .../server/dao/sql/event/EventRepository.java | 384 ++++++++---------- .../server/dao/sql/event/JpaBaseEventDao.java | 148 ++++--- .../sql/event/LifecycleEventRepository.java | 76 ++++ .../sql/event/StatisticsEventRepository.java | 46 +++ .../sql/schema-entities-idx-psql-addon.sql | 24 +- .../main/resources/sql/schema-entities.sql | 12 +- .../dao/service/AbstractServiceTest.java | 2 +- .../service/event/BaseEventServiceTest.java | 11 +- .../dao/sql/event/JpaBaseEventDaoTest.java | 20 - 34 files changed, 923 insertions(+), 540 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index e6f207d960..b4c308db86 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -488,7 +488,7 @@ public class ActorSystemContext { public void persistError(TenantId tenantId, EntityId entityId, String method, Exception e) { eventService.saveAsync(ErrorEvent.builder() .tenantId(tenantId) - .entityId(entityId) + .entityId(entityId.getId()) .serviceId(getServiceId()) .method(method) .error(toString(e)).build()); @@ -497,14 +497,14 @@ public class ActorSystemContext { public void persistLifecycleEvent(TenantId tenantId, EntityId entityId, ComponentLifecycleEvent lcEvent, Exception e) { LifecycleEvent.LifecycleEventBuilder event = LifecycleEvent.builder() .tenantId(tenantId) - .entityId(entityId) + .entityId(entityId.getId()) .serviceId(getServiceId()) .lcEventType(lcEvent.name()); if (e != null) { event.success(false).error(toString(e)); } else { - event.success(false); + event.success(true); } eventService.saveAsync(event.build()); @@ -553,7 +553,7 @@ public class ActorSystemContext { try { RuleNodeDebugEvent.RuleNodeDebugEventBuilder event = RuleNodeDebugEvent.builder() .tenantId(tenantId) - .entityId(entityId) + .entityId(entityId.getId()) .serviceId(getServiceId()) .eventType(type) .eventEntity(tbMsg.getOriginator()) @@ -600,7 +600,7 @@ public class ActorSystemContext { private void persistRuleChainDebugModeEvent(TenantId tenantId, EntityId entityId, Throwable error) { RuleChainDebugEvent.RuleChainDebugEventBuilder event = RuleChainDebugEvent.builder() .tenantId(tenantId) - .entityId(entityId) + .entityId(entityId.getId()) .serviceId(getServiceId()) .message("Reached debug mode rate limit!"); if (error != null) { diff --git a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java index fa5dedf66f..247a6c62d0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/stats/StatsActor.java @@ -56,7 +56,7 @@ public class StatsActor extends ContextAwareActor { } systemContext.getEventService().saveAsync(StatisticsEvent.builder() .tenantId(msg.getTenantId()) - .entityId(msg.getEntityId()) + .entityId(msg.getEntityId().getId()) .serviceId(systemContext.getServiceInfoProvider().getServiceId()) .messagesProcessed(msg.getMessagesProcessed()) .errorsOccurred(msg.getErrorsOccurred()) diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index bdf74c9055..d4199006d0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -94,7 +94,7 @@ public class ControllerConstants { protected static final String DEVICE_PROFILE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, transportType, description, isDefault"; protected static final String ASSET_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle"; protected static final String ALARM_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, startTs, endTs, type, ackTs, clearTs, severity, status"; - protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, id"; + protected static final String EVENT_SORT_PROPERTY_ALLOWABLE_VALUES = "ts, id"; protected static final String EDGE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, type, label, customerTitle"; protected static final String RULE_CHAIN_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, name, root"; protected static final String WIDGET_BUNDLE_SORT_PROPERTY_ALLOWABLE_VALUES = "createdTime, title, tenantId"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EventController.java b/application/src/main/java/org/thingsboard/server/controller/EventController.java index dd6c05749f..45042dc883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EventController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EventController.java @@ -29,8 +29,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -42,6 +45,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; +import java.util.Locale; + import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE; @@ -135,16 +140,12 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEvents(tenantId, entityId, eventType, pageLink)); - } catch (Exception e) { - throw handleException(e); - } + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + return checkNotNull(eventService.findEvents(tenantId, entityId, resolveEventType(eventType), pageLink)); } @ApiOperation(value = "Get Events (getEvents)", @@ -176,18 +177,14 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEvents(tenantId, entityId, pageLink)); - } catch (Exception e) { - throw handleException(e); - } + return checkNotNull(eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, pageLink)); } @ApiOperation(value = "Get Events by event filter (getEvents)", @@ -223,21 +220,17 @@ public class EventController extends BaseController { @RequestParam(required = false) Long endTime) throws ThingsboardException { checkParameter("EntityId", strEntityId); checkParameter("EntityType", strEntityType); - try { - TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - - EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); - checkEntityId(entityId, Operation.READ); + TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); - if (sortProperty != null && sortProperty.equals("createdTime") && eventFilter.hasFilterForJsonBody()) { - sortProperty = ModelConstants.CREATED_TIME_PROPERTY; - } + EntityId entityId = EntityIdFactory.getByTypeAndId(strEntityType, strEntityId); + checkEntityId(entityId, Operation.READ); - TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); - return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink)); - } catch (Exception e) { - throw handleException(e); + if (sortProperty != null && sortProperty.equals("createdTime")) { + sortProperty = ModelConstants.TS_COLUMN; } + + TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); + return checkNotNull(eventService.findEventsByFilter(tenantId, entityId, eventFilter, pageLink)); } @ApiOperation(value = "Clear Events (clearEvents)", notes = "Clears events by filter for specified entity.") @@ -266,4 +259,13 @@ public class EventController extends BaseController { } } + private static EventType resolveEventType(String eventType) throws ThingsboardException { + for (var et : EventType.values()) { + if (et.name().equalsIgnoreCase(eventType) || et.getOldName().equalsIgnoreCase(eventType)) { + return et; + } + } + throw new ThingsboardException("Event type: '" + eventType + "' is not supported!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 143844504a..bcca7965b3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -352,7 +353,7 @@ public class RuleChainController extends BaseController { RuleNodeId ruleNodeId = new RuleNodeId(toUUID(strRuleNodeId)); checkRuleNode(ruleNodeId, Operation.READ); TenantId tenantId = getCurrentUser().getTenantId(); - List events = eventService.findLatestEvents(tenantId, ruleNodeId, DataConstants.DEBUG_RULE_NODE, 2); + List events = eventService.findLatestEvents(tenantId, ruleNodeId, EventType.DEBUG_RULE_NODE, 2); JsonNode result = null; if (events != null) { for (EventInfo event : events) { diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index 7335c10d13..ca0d5a6215 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -61,7 +62,7 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle } protected PageData getDebugEvents(TenantId tenantId, EntityId entityId, int limit) throws Exception { - return getEvents(tenantId, entityId, DataConstants.DEBUG_RULE_NODE, limit); + return getEvents(tenantId, entityId, EventType.DEBUG_RULE_NODE.getOldName(), limit); } protected PageData getEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) throws Exception { @@ -72,7 +73,6 @@ public abstract class AbstractRuleEngineControllerTest extends AbstractControlle } - protected JsonNode getMetadata(EventInfo outEvent) { String metaDataStr = outEvent.getBody().get("metadata").asText(); try { diff --git a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java index c840cf4c9a..70cd292e00 100644 --- a/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/lifecycle/AbstractRuleEngineLifecycleIntegrationTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.rule.RuleChain; @@ -109,7 +110,7 @@ public abstract class AbstractRuleEngineLifecycleIntegrationTest extends Abstrac .pollInterval(10, MILLISECONDS) .atMost(TIMEOUT, TimeUnit.SECONDS) .until(() -> { - List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), DataConstants.LC_EVENT, 1000) + List debugEvents = getEvents(tenantId, ruleChainFinal.getFirstRuleNodeId(), EventType.LC_EVENT.getOldName(), 1000) .getData().stream().filter(e -> { var body = e.getBody(); return body.has("event") && body.get("event").asText().equals("STARTED") diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 0d157e755d..b5f4c98e25 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -31,13 +32,9 @@ public interface EventService { ListenableFuture saveAsync(Event event); - Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid); + PageData findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink); - PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink); - - PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink); - - List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); + List findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit); PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 1351621bf5..90d95fb561 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -52,12 +52,6 @@ public class DataConstants { } public static final String ALARM = "ALARM"; - public static final String ERROR = "ERROR"; - public static final String LC_EVENT = "LC_EVENT"; - public static final String STATS = "STATS"; - public static final String DEBUG_RULE_NODE = "DEBUG_RULE_NODE"; - public static final String DEBUG_RULE_CHAIN = "DEBUG_RULE_CHAIN"; - public static final String IN = "IN"; public static final String OUT = "OUT"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java index 8e06177ca6..f811a118cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java @@ -20,9 +20,13 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.UUID; + @ToString @EqualsAndHashCode(callSuper = true) public class ErrorEvent extends Event { @@ -30,8 +34,8 @@ public class ErrorEvent extends Event { private static final long serialVersionUID = 960461434033192571L; @Builder - private ErrorEvent(TenantId tenantId, EntityId entityId, String serviceId, String method, String error) { - super(tenantId, entityId, serviceId); + private ErrorEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String method, String error) { + super(tenantId, entityId, serviceId, id, ts); this.method = method; this.error = error; } @@ -45,4 +49,9 @@ public class ErrorEvent extends Event { public EventType getType() { return EventType.ERROR; } + + @Override + public EventInfo toInfo(EntityType entityType) { + return null; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java index 9c013e1486..6e41a3750e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java @@ -15,28 +15,49 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.UUID; + @Data @EqualsAndHashCode(callSuper = true) public abstract class Event extends BaseData { - private final TenantId tenantId; - private final EntityId entityId; - private final String serviceId; + protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + protected final TenantId tenantId; + protected final UUID entityId; + protected final String serviceId; - public Event(TenantId tenantId, EntityId entityId, String serviceId) { + public Event(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts) { super(); + if (id != null) { + this.id = new EventId(id); + } this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID; this.entityId = entityId; this.serviceId = serviceId; + this.createdTime = ts; } public abstract EventType getType(); + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = new EventInfo(); + eventInfo.setTenantId(tenantId); + eventInfo.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + eventInfo.setType(getType().getOldName()); + eventInfo.setId(id); + eventInfo.setCreatedTime(createdTime); + eventInfo.setBody(OBJECT_MAPPER.createObjectNode().put("server", getServiceId())); + return eventInfo; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 8d58158521..5fa6fb7370 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -18,13 +18,16 @@ package org.thingsboard.server.common.data.event; import lombok.Getter; public enum EventType { - ERROR("error_event"), LC_EVENT("lc_event"), STATS("stats_event"), DEBUG_RULE_NODE("rule_node_debug_event"), DEBUG_RULE_CHAIN("rule_chain_debug_event"); + ERROR("error_event", "ERROR"), LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE"), DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN"); @Getter private final String table; + @Getter + private final String oldName; - EventType(String table) { + EventType(String table, String oldName) { this.table = table; + this.oldName = oldName; } } \ No newline at end of file diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java index 217955af96..1e78df36a6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java @@ -15,14 +15,21 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.jetbrains.annotations.NotNull; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import java.util.UUID; + @ToString @EqualsAndHashCode(callSuper = true) public class LifecycleEvent extends Event { @@ -30,8 +37,10 @@ public class LifecycleEvent extends Event { private static final long serialVersionUID = -3247420461850911549L; @Builder - private LifecycleEvent(TenantId tenantId, EntityId entityId, String serviceId, String lcEventType, boolean success, String error) { - super(tenantId, entityId, serviceId); + private LifecycleEvent(TenantId tenantId, UUID entityId, String serviceId, + UUID id, long ts, + String lcEventType, boolean success, String error) { + super(tenantId, entityId, serviceId, id, ts); this.lcEventType = lcEventType; this.success = success; this.error = error; @@ -41,11 +50,25 @@ public class LifecycleEvent extends Event { private final String lcEventType; @Getter private final boolean success; - @Getter @Setter + @Getter + @Setter private String error; @Override public EventType getType() { return EventType.LC_EVENT; } + + @Override + public EventInfo toInfo(EntityType entityType) { + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("event", lcEventType) + .put("success", success); + if (error != null) { + json.put("error", error); + } + return eventInfo; + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java index ac8c16c2e5..52c961c1c8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java @@ -20,6 +20,8 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,8 +34,8 @@ public class RuleChainDebugEvent extends Event { private static final long serialVersionUID = -386392236201116767L; @Builder - private RuleChainDebugEvent(TenantId tenantId, EntityId entityId, String serviceId, String message, String error) { - super(tenantId, entityId, serviceId); + private RuleChainDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String message, String error) { + super(tenantId, entityId, serviceId, id, ts); this.message = message; this.error = error; } @@ -47,4 +49,9 @@ public class RuleChainDebugEvent extends Event { public EventType getType() { return EventType.DEBUG_RULE_CHAIN; } + + @Override + public EventInfo toInfo(EntityType entityType) { + return null; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java index f9280d2d6b..891ece5f22 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -20,6 +20,8 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,11 +34,11 @@ public class RuleNodeDebugEvent extends Event { private static final long serialVersionUID = -6575797430064573984L; @Builder - private RuleNodeDebugEvent(TenantId tenantId, EntityId entityId, String serviceId, + private RuleNodeDebugEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, String eventType, EntityId eventEntity, UUID msgId, String msgType, String dataType, String relationType, String data, String metadata, String error) { - super(tenantId, entityId, serviceId); + super(tenantId, entityId, serviceId, id, ts); this.eventType = eventType; this.eventEntity = eventEntity; this.msgId = msgId; @@ -72,4 +74,9 @@ public class RuleNodeDebugEvent extends Event { public EventType getType() { return EventType.DEBUG_RULE_NODE; } + + @Override + public EventInfo toInfo(EntityType entityType) { + return null; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java index b0194762de..b574949cf8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java @@ -19,9 +19,13 @@ import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.util.UUID; + @ToString @EqualsAndHashCode(callSuper = true) public class StatisticsEvent extends Event { @@ -29,8 +33,8 @@ public class StatisticsEvent extends Event { private static final long serialVersionUID = 6683733979448910631L; @Builder - private StatisticsEvent(TenantId tenantId, EntityId entityId, String serviceId, long messagesProcessed, long errorsOccurred) { - super(tenantId, entityId, serviceId); + private StatisticsEvent(TenantId tenantId, UUID entityId, String serviceId, UUID id, long ts, long messagesProcessed, long errorsOccurred) { + super(tenantId, entityId, serviceId, id, ts); this.messagesProcessed = messagesProcessed; this.errorsOccurred = errorsOccurred; } @@ -44,4 +48,9 @@ public class StatisticsEvent extends Event { public EventType getType() { return EventType.STATS; } + + @Override + public EventInfo toInfo(EntityType entityType) { + return null; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index d7f0885bdc..311af9af44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -20,24 +20,23 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import java.util.List; -import java.util.Optional; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -97,41 +96,18 @@ public class BaseEventService implements EventService { } @Override - public Optional findEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) { - if (tenantId == null) { - throw new DataValidationException("Tenant id should be specified!."); - } - if (entityId == null) { - throw new DataValidationException("Entity id should be specified!."); - } - if (StringUtils.isEmpty(eventType)) { - throw new DataValidationException("Event type should be specified!."); - } - if (StringUtils.isEmpty(eventUid)) { - throw new DataValidationException("Event uid should be specified!."); - } - EventInfo event = eventDao.findEvent(tenantId.getId(), entityId, eventType, eventUid); - return event != null ? Optional.of(event) : Optional.empty(); - } - - @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, TimePageLink pageLink) { - return eventDao.findEvents(tenantId.getId(), entityId, pageLink); + public PageData findEvents(TenantId tenantId, EntityId entityId, EventType eventType, TimePageLink pageLink) { + return convert(entityId.getEntityType(), eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, pageLink)); } @Override - public PageData findEvents(TenantId tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { - return eventDao.findEvents(tenantId.getId(), entityId, eventType, pageLink); - } - - @Override - public List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit) { - return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit); + public List findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit) { + return eventDao.findLatestEvents(tenantId.getId(), entityId.getId(), eventType, limit); } @Override public PageData findEventsByFilter(TenantId tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { - return eventDao.findEventByFilter(tenantId.getId(), entityId, eventFilter, pageLink); + return convert(entityId.getEntityType(), eventDao.findEventByFilter(tenantId.getId(), entityId.getId(), eventFilter, pageLink)); } @Override @@ -141,23 +117,29 @@ public class BaseEventService implements EventService { @Override public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) { - TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime); - PageData eventsPageData; - do { - if (eventFilter == null) { - eventsPageData = findEvents(tenantId, entityId, eventsPageLink); - } else { - eventsPageData = findEventsByFilter(tenantId, entityId, eventFilter, eventsPageLink); - } - - eventDao.removeAllByIds(eventsPageData.getData().stream() - .map(IdBased::getUuidId) - .collect(Collectors.toList())); - } while (eventsPageData.hasNext()); +// TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime); +// PageData eventsPageData; +// do { +// if (eventFilter == null) { +// eventsPageData = findEvents(tenantId, entityId, eventsPageLink); +// } else { +// eventsPageData = findEventsByFilter(tenantId, entityId, eventFilter, eventsPageLink); +// } +// +// eventDao.removeAllByIds(eventsPageData.getData().stream() +// .map(IdBased::getUuidId) +// .collect(Collectors.toList())); +// } while (eventsPageData.hasNext()); } @Override public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { eventDao.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); } + + private PageData convert(EntityType entityType, PageData pd) { + return new PageData<>(pd.getData() == null ? null : + pd.getData().stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()) + , pd.getTotalPages(), pd.getTotalElements(), pd.hasNext()); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 30815bd15e..1878594fb4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; @@ -30,7 +31,7 @@ import java.util.UUID; /** * The Interface EventDao. */ -public interface EventDao extends Dao { +public interface EventDao { /** * Save or update event object async @@ -40,17 +41,6 @@ public interface EventDao extends Dao { */ ListenableFuture saveAsync(Event event); - /** - * Find event by tenantId, entityId and eventUid. - * - * @param tenantId the tenantId - * @param entityId the entityId - * @param eventType the eventType - * @param eventUid the eventUid - * @return the event - */ - EventInfo findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid); - /** * Find events by tenantId, entityId and pageLink. * @@ -70,9 +60,9 @@ public interface EventDao extends Dao { * @param pageLink the pageLink * @return the event list */ - PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink); + PageData findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink); - PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink); + PageData findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink); /** * Find latest events by tenantId, entityId and eventType. @@ -83,7 +73,7 @@ public interface EventDao extends Dao { * @param limit the limit * @return the event list */ - List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit); + List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit); /** * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 0b4a65f43c..37934ed306 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -368,16 +368,25 @@ public class ModelConstants { /** * Cassandra event constants. */ - public static final String EVENT_COLUMN_FAMILY_NAME = "event"; + public static final String ERROR_EVENT_TABLE_NAME = "error_event"; + public static final String LC_EVENT_TABLE_NAME = "lc_event"; + public static final String STATS_EVENT_TABLE_NAME = "stats_event"; + public static final String RULE_NODE_DEBUG_EVENT_TABLE_NAME = "rule_node_debug_event"; + public static final String RULE_CHAIN_DEBUG_EVENT_TABLE_NAME = "rule_chain_debug_event"; + public static final String EVENT_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; - public static final String EVENT_TYPE_PROPERTY = "event_type"; - public static final String EVENT_UID_PROPERTY = "event_uid"; - public static final String EVENT_ENTITY_TYPE_PROPERTY = ENTITY_TYPE_PROPERTY; + public static final String EVENT_SERVICE_ID_PROPERTY = "service_id"; public static final String EVENT_ENTITY_ID_PROPERTY = "entity_id"; public static final String EVENT_BODY_PROPERTY = "body"; - public static final String EVENT_BY_TYPE_AND_ID_VIEW_NAME = "event_by_type_and_id"; - public static final String EVENT_BY_ID_VIEW_NAME = "event_by_id"; + public static final String EVENT_MESSAGES_PROCESSED_COLUMN_NAME = "e_messages_processed"; + public static final String EVENT_ERRORS_OCCURRED_COLUMN_NAME = "e_errors_occurred"; + + public static final String EVENT_METHOD_COLUMN_NAME = "e_method"; + + public static final String EVENT_TYPE_COLUMN_NAME = "e_type"; + public static final String EVENT_ERROR_COLUMN_NAME = "e_error"; + public static final String EVENT_SUCCESS_COLUMN_NAME = "e_success"; public static final String DEBUG_MODE = "debug_mode"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java new file mode 100644 index 0000000000..397824140f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.TypeDef; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.util.mapping.JsonStringType; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.ERROR_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METHOD_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@TypeDef(name = "json", typeClass = JsonStringType.class) +@Table(name = ERROR_EVENT_TABLE_NAME) +@NoArgsConstructor +public class ErrorEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_METHOD_COLUMN_NAME) + private String method; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public ErrorEventEntity(ErrorEvent event) { + super(event); + this.method = event.getMethod(); + this.error = event.getError(); + } + + @Override + public ErrorEvent toData() { + return ErrorEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .method(method) + .error(error) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index bf08333e7b..b0a0135849 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -15,104 +15,80 @@ */ package org.thingsboard.server.dao.model.sql; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.EventId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Table; +import javax.persistence.Id; +import javax.persistence.MappedSuperclass; import java.util.UUID; -import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SERVICE_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data -@EqualsAndHashCode(callSuper = true) -@Entity @TypeDef(name = "json", typeClass = JsonStringType.class) -@Table(name = EVENT_COLUMN_FAMILY_NAME) @NoArgsConstructor -public class EventEntity extends BaseSqlEntity implements BaseEntity { +@MappedSuperclass +public abstract class EventEntity implements BaseEntity { - @Column(name = EVENT_TENANT_ID_PROPERTY) - private UUID tenantId; + @Id + @Column(name = ModelConstants.ID_PROPERTY, columnDefinition = "uuid") + protected UUID id; - @Enumerated(EnumType.STRING) - @Column(name = EVENT_ENTITY_TYPE_PROPERTY) - private EntityType entityType; + @Column(name = EVENT_TENANT_ID_PROPERTY, columnDefinition = "uuid") + protected UUID tenantId; - @Column(name = EVENT_ENTITY_ID_PROPERTY) - private UUID entityId; + @Column(name = EVENT_ENTITY_ID_PROPERTY, columnDefinition = "uuid") + protected UUID entityId; - @Column(name = EVENT_TYPE_PROPERTY) - private String eventType; + @Column(name = EVENT_SERVICE_ID_PROPERTY) + protected String serviceId; - @Column(name = EVENT_UID_PROPERTY) - private String eventUid; + @Column(name = TS_COLUMN) + protected long ts; - @Type(type = "json") - @Column(name = EVENT_BODY_PROPERTY) - private JsonNode body; + public EventEntity(UUID id, UUID tenantId, UUID entityId, String serviceId, long ts) { + this.id = id; + this.tenantId = tenantId; + this.entityId = entityId; + this.serviceId = serviceId; + this.ts = ts; + } - @Column(name = TS_COLUMN) - private long ts; - - public EventEntity(EventInfo event) { - if (event.getId() != null) { - this.setUuid(event.getId().getId()); - this.ts = getTs(event.getId().getId()); - } else { - this.ts = System.currentTimeMillis(); - } - this.setCreatedTime(event.getCreatedTime()); - if (event.getTenantId() != null) { - this.tenantId = event.getTenantId().getId(); - } - if (event.getEntityId() != null) { - this.entityType = event.getEntityId().getEntityType(); - this.entityId = event.getEntityId().getId(); - } - this.eventType = event.getType(); - this.eventUid = event.getUid(); - this.body = event.getBody(); + public EventEntity(Event event) { + this.id = event.getId().getId(); + this.tenantId = event.getTenantId().getId(); + this.entityId = event.getEntityId(); + this.serviceId = event.getServiceId(); + this.ts = event.getCreatedTime(); + } + + @Override + public UUID getUuid() { + return id; } + @Override + public void setUuid(UUID id) { + this.id = id; + } @Override - public Event toData() { - EventInfo event = new EventInfo(new EventId(this.getUuid())); - event.setCreatedTime(createdTime); - event.setTenantId(TenantId.fromUUID(tenantId)); - event.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - event.setBody(body); - event.setType(eventType); - event.setUid(eventUid); - return null; + public long getCreatedTime() { + return ts; } - private static long getTs(UUID uuid) { - return (uuid.timestamp() - EPOCH_DIFF) / 10000; + @Override + public void setCreatedTime(long createdTime) { + ts = createdTime; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java new file mode 100644 index 0000000000..0a4380cb63 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.TypeDef; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.util.mapping.JsonStringType; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@TypeDef(name = "json", typeClass = JsonStringType.class) +@Table(name = LC_EVENT_TABLE_NAME) +@NoArgsConstructor +public class LifecycleEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_TYPE_COLUMN_NAME) + private String eventType; + @Column(name = EVENT_SUCCESS_COLUMN_NAME) + private boolean success; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public LifecycleEventEntity(LifecycleEvent event) { + super(event); + this.eventType = event.getLcEventType(); + this.success = event.isSuccess(); + this.error = event.getError(); + } + + @Override + public LifecycleEvent toData() { + return LifecycleEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .lcEventType(eventType) + .success(success) + .error(error) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java new file mode 100644 index 0000000000..240e6fdcf7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.hibernate.annotations.TypeDef; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.util.mapping.JsonStringType; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERRORS_OCCURRED_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGES_PROCESSED_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@TypeDef(name = "json", typeClass = JsonStringType.class) +@Table(name = STATS_EVENT_TABLE_NAME) +@NoArgsConstructor +public class StatisticsEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_MESSAGES_PROCESSED_COLUMN_NAME) + private long messagesProcessed; + @Column(name = EVENT_ERRORS_OCCURRED_COLUMN_NAME) + private long errorsOccurred; + + public StatisticsEventEntity(StatisticsEvent event) { + super(event); + this.messagesProcessed = event.getMessagesProcessed(); + this.errorsOccurred = event.getErrorsOccurred(); + } + + @Override + public StatisticsEvent toData() { + return StatisticsEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .messagesProcessed(messagesProcessed) + .errorsOccurred(errorsOccurred) + .build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java new file mode 100644 index 0000000000..721ffe42fe --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.model.sql.ErrorEventEntity; +import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; + +import java.util.UUID; + +/** + * Created by Valerii Sosliuk on 5/3/2017. + */ +public interface ErrorEventRepository extends JpaRepository { + + @Query("SELECT e FROM ErrorEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("method") String method, + @Param("error") String error, + Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index e69b8578df..79d7d250e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -73,7 +73,7 @@ public class EventInsertRepository { " (id, tenant_id, ts, entity_id, service_id, e_type, e_success, e_error) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); insertStmtMap.put(EventType.STATS, "INSERT INTO " + EventType.STATS.getTable() + - " (id, tenant_id, ts, entity_id, service_id, e_messages_processed, e_errors_occured) " + + " (id, tenant_id, ts, entity_id, service_id, e_messages_processed, e_errors_occurred) " + "VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING;"); insertStmtMap.put(EventType.DEBUG_RULE_NODE, "INSERT INTO " + EventType.DEBUG_RULE_NODE.getTable() + " (id, tenant_id, ts, entity_id, service_id, e_type, e_entity_id, e_entity_type, e_msg_id, e_msg_type, e_data_type, e_relation_type, e_data, e_metadata, e_error) " + @@ -171,8 +171,8 @@ public class EventInsertRepository { RuleNodeDebugEvent event = (RuleNodeDebugEvent) events.get(i); setCommonEventFields(ps, event); safePutString(ps, 6, event.getEventType()); - safePutUUID(ps, 7, event.getEntityId() != null ? event.getEntityId().getId() : null); - safePutString(ps, 8, event.getEntityId() != null ? event.getEntityId().getEntityType().name() : null); + safePutUUID(ps, 7, event.getEventEntity() != null ? event.getEventEntity().getId() : null); + safePutString(ps, 8, event.getEventEntity() != null ? event.getEventEntity().getEntityType().name() : null); safePutUUID(ps, 9, event.getMsgId()); safePutString(ps, 10, event.getMsgType()); safePutString(ps, 11, event.getDataType()); @@ -226,7 +226,7 @@ public class EventInsertRepository { ps.setObject(1, event.getId().getId()); ps.setObject(2, event.getTenantId().getId()); ps.setLong(3, event.getCreatedTime()); - ps.setObject(4, event.getEntityId().getId()); + ps.setObject(4, event.getEntityId()); ps.setString(5, event.getServiceId()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index e36b82f1e8..8a967395ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -31,214 +31,178 @@ import java.util.UUID; */ public interface EventRepository extends JpaRepository { - EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, - EntityType entityType, - UUID entityId, - String eventType, - String eventUid); - - EventEntity findByTenantIdAndEntityTypeAndEntityId(UUID tenantId, - EntityType entityType, - UUID entityId); - - @Query("SELECT e FROM EventEntity e WHERE e.tenantId = :tenantId AND e.entityType = :entityType " + - "AND e.entityId = :entityId AND e.eventType = :eventType ORDER BY e.eventType DESC, e.id DESC") - List findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( - @Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("eventType") String eventType, - Pageable pageable); - - @Query("SELECT e FROM EventEntity e WHERE " + - "e.tenantId = :tenantId " + - "AND e.entityType = :entityType AND e.entityId = :entityId " + - "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR e.createdTime <= :endTime) " + - "AND LOWER(e.eventType) LIKE LOWER(CONCAT('%', :textSearch, '%'))" - ) - Page findEventsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("textSearch") String textSearch, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); - - @Query("SELECT e FROM EventEntity e WHERE " + - "e.tenantId = :tenantId " + - "AND e.entityType = :entityType AND e.entityId = :entityId " + - "AND e.eventType = :eventType " + - "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + - "AND (:endTime IS NULL OR e.createdTime <= :endTime)" - ) - Page findEventsByTenantIdAndEntityIdAndEventType(@Param("tenantId") UUID tenantId, - @Param("entityType") EntityType entityType, - @Param("entityId") UUID entityId, - @Param("eventType") String eventType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); - - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = :eventType " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + - "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + - "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + - "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + - "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + - "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + - "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + - "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%')) ", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = :eventType " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + - "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + - "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + - "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + - "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + - "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + - "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + - "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%'))" - ) - Page findDebugRuleNodeEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("eventType") String eventType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - @Param("type") String type, - @Param("server") String server, - @Param("entityName") String entityName, - @Param("relationType") String relationType, - @Param("bodyEntityId") String bodyEntityId, - @Param("msgType") String msgType, - @Param("isError") boolean isError, - @Param("error") String error, - @Param("data") String data, - @Param("metadata") String metadata, - Pageable pageable); - - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'ERROR' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'ERROR' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))") - Page findErrorEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("method") String method, - @Param("error") String error, - Pageable pageable); - - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " + - "AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))" - , - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:event IS NULL OR lower(json_body->>'event') LIKE concat('%', lower(:event\\:\\:varchar), '%')) " + - "AND ((:statusFilterEnabled = FALSE) OR lower(json_body->>'success')\\:\\:boolean = :statusFilter) " + - "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))" - ) - Page findLifeCycleEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("event") String event, - @Param("statusFilterEnabled") boolean statusFilterEnabled, - @Param("statusFilter") boolean statusFilter, - @Param("error") String error, - Pageable pageable); - - @Query(nativeQuery = true, - value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'STATS' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + - "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ", - countQuery = "SELECT count(*) FROM " + - "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + - "e.tenant_id = :tenantId " + - "AND e.entity_type = :entityType " + - "AND e.entity_id = :entityId " + - "AND e.event_type = 'LC_EVENT' " + - "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + - ") AS e WHERE " + - "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + - "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + - "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ") - Page findStatisticsEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("entityType") String entityType, - @Param("startTime") Long startTime, - @Param("endTime") Long endTIme, - @Param("server") String server, - @Param("messagesProcessed") Integer messagesProcessed, - @Param("errorsOccurred") Integer errorsOccurred, - Pageable pageable); - +// EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, +// EntityType entityType, +// UUID entityId, +// String eventType, +// String eventUid); +// +// EventEntity findByTenantIdAndEntityTypeAndEntityId(UUID tenantId, +// EntityType entityType, +// UUID entityId); +// +// @Query("SELECT e FROM EventEntity e WHERE e.tenantId = :tenantId AND e.entityType = :entityType " + +// "AND e.entityId = :entityId AND e.eventType = :eventType ORDER BY e.eventType DESC, e.id DESC") +// List findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( +// @Param("tenantId") UUID tenantId, +// @Param("entityType") EntityType entityType, +// @Param("entityId") UUID entityId, +// @Param("eventType") String eventType, +// Pageable pageable); +// +// @Query("SELECT e FROM EventEntity e WHERE " + +// "e.tenantId = :tenantId " + +// "AND e.entityType = :entityType AND e.entityId = :entityId " + +// "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + +// "AND (:endTime IS NULL OR e.createdTime <= :endTime) " + +// "AND LOWER(e.eventType) LIKE LOWER(CONCAT('%', :textSearch, '%'))" +// ) +// Page findEventsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId, +// @Param("entityType") EntityType entityType, +// @Param("entityId") UUID entityId, +// @Param("textSearch") String textSearch, +// @Param("startTime") Long startTime, +// @Param("endTime") Long endTime, +// Pageable pageable); +// +// +// +// @Query("SELECT e FROM EventEntity e WHERE " + +// "e.tenantId = :tenantId " + +// "AND e.entityType = :entityType AND e.entityId = :entityId " + +// "AND e.eventType = :eventType " + +// "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + +// "AND (:endTime IS NULL OR e.createdTime <= :endTime)" +// ) +// Page findEventsByTenantIdAndEntityIdAndEventType(@Param("tenantId") UUID tenantId, +// @Param("entityType") EntityType entityType, +// @Param("entityId") UUID entityId, +// @Param("eventType") String eventType, +// @Param("startTime") Long startTime, +// @Param("endTime") Long endTime, +// Pageable pageable); +// +// @Query(nativeQuery = true, +// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = :eventType " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + +// "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + +// "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + +// "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + +// "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + +// "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + +// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + +// "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + +// "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%')) ", +// countQuery = "SELECT count(*) FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = :eventType " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + +// "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + +// "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + +// "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + +// "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + +// "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + +// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + +// "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + +// "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%'))" +// ) +// Page findDebugRuleNodeEvents(@Param("tenantId") UUID tenantId, +// @Param("entityId") UUID entityId, +// @Param("entityType") String entityType, +// @Param("eventType") String eventType, +// @Param("startTime") Long startTime, +// @Param("endTime") Long endTime, +// @Param("type") String type, +// @Param("server") String server, +// @Param("entityName") String entityName, +// @Param("relationType") String relationType, +// @Param("bodyEntityId") String bodyEntityId, +// @Param("msgType") String msgType, +// @Param("isError") boolean isError, +// @Param("error") String error, +// @Param("data") String data, +// @Param("metadata") String metadata, +// Pageable pageable); +// +// @Query(nativeQuery = true, +// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = 'ERROR' " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + +// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))", +// countQuery = "SELECT count(*) FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = 'ERROR' " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + +// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))") +// Page findErrorEvents(@Param("tenantId") UUID tenantId, +// @Param("entityId") UUID entityId, +// @Param("entityType") String entityType, +// @Param("startTime") Long startTime, +// @Param("endTime") Long endTIme, +// @Param("server") String server, +// @Param("method") String method, +// @Param("error") String error, +// Pageable pageable); +// +// +// @Query(nativeQuery = true, +// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = 'STATS' " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + +// "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ", +// countQuery = "SELECT count(*) FROM " + +// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + +// "e.tenant_id = :tenantId " + +// "AND e.entity_type = :entityType " + +// "AND e.entity_id = :entityId " + +// "AND e.event_type = 'LC_EVENT' " + +// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + +// ") AS e WHERE " + +// "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + +// "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + +// "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ") +// Page findStatisticsEvents(@Param("tenantId") UUID tenantId, +// @Param("entityId") UUID entityId, +// @Param("entityType") String entityType, +// @Param("startTime") Long startTime, +// @Param("endTime") Long endTIme, +// @Param("server") String server, +// @Param("messagesProcessed") Integer messagesProcessed, +// @Param("errorsOccurred") Integer errorsOccurred, +// Pageable pageable); +// } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index deb3c87402..0717fa6896 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -18,21 +18,22 @@ package org.thingsboard.server.dao.sql.event; import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DataIntegrityViolationException; -import org.springframework.data.domain.PageRequest; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.event.DebugEvent; +import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifeCycleEventFilter; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.event.StatisticsEventFilter; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; @@ -41,22 +42,14 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.event.EventDao; -import org.thingsboard.server.dao.model.sql.EventEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; import org.thingsboard.server.dao.timeseries.SqlPartition; -import org.thingsboard.server.dao.timeseries.SqlTsPartitionDate; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -72,7 +65,7 @@ import java.util.function.Function; */ @Slf4j @Component -public class JpaBaseEventDao extends JpaAbstractDao implements EventDao { +public class JpaBaseEventDao implements EventDao { private static final long PARTITION_DURATION = TimeUnit.HOURS.toMillis(1); private final Map> partitionsByEventType = new ConcurrentHashMap<>(); @@ -85,20 +78,19 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen private EventRepository eventRepository; @Autowired - private EventInsertRepository eventInsertRepository; + private LifecycleEventRepository lcEventRepository; @Autowired - private EventCleanupRepository eventCleanupRepository; + private StatisticsEventRepository statsEventRepository; - @Override - protected Class getEntityClass() { - return EventEntity.class; - } + @Autowired + private ErrorEventRepository errorEventRepository; - @Override - protected JpaRepository getRepository() { - return eventRepository; - } + @Autowired + private EventInsertRepository eventInsertRepository; + + @Autowired + private EventCleanupRepository eventCleanupRepository; @Autowired ScheduledLogExecutorComponent logExecutor; @@ -198,13 +190,6 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen } } - @Override - public EventInfo findEvent(UUID tenantId, EntityId entityId, String eventType, String eventUid) { - return null; -// return DaoUtil.getData(eventRepository.findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid( -// tenantId, entityId.getEntityType(), entityId.getId(), eventType, eventUid)); - } - @Override public PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) { return null; @@ -221,22 +206,37 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen } @Override - public PageData findEvents(UUID tenantId, EntityId entityId, String eventType, TimePageLink pageLink) { - return null; -// return DaoUtil.toPageData( -// eventRepository -// .findEventsByTenantIdAndEntityIdAndEventType( -// tenantId, -// entityId.getEntityType(), -// entityId.getId(), -// eventType, -// pageLink.getStartTime(), -// pageLink.getEndTime(), -// DaoUtil.toPageable(pageLink))); + public PageData findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink) { + switch (eventType) { + case LC_EVENT: + return findLcEventsWithoutFilter(tenantId, entityId, pageLink); + case STATS: + return findStatsEventsWithoutFilter(tenantId, entityId, pageLink); + case ERROR: + return findErrorEventsWithoutFilter(tenantId, entityId, pageLink); + default: + throw new RuntimeException("Event type: " + eventType + " is not supported!"); + } + } + + private PageData findLcEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { + return DaoUtil.toPageData( + lcEventRepository.findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); } + private PageData findStatsEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { + return DaoUtil.toPageData( + statsEventRepository.findEventsWithoutFilter(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); + } + + private PageData findErrorEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { + return DaoUtil.toPageData( + errorEventRepository.findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); + } + + @Override - public PageData findEventByFilter(UUID tenantId, EntityId entityId, EventFilter eventFilter, TimePageLink pageLink) { + public PageData findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink) { if (eventFilter.hasFilterForJsonBody()) { switch (eventFilter.getEventType()) { case DEBUG_RULE_NODE: @@ -252,7 +252,7 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); } } else { - return findEvents(tenantId, entityId, eventFilter.getEventType().name(), pageLink); + return findEvents(tenantId, entityId, eventFilter.getEventType(), pageLink); } } @@ -279,40 +279,36 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen // DaoUtil.toPageable(pageLink))); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { - return null; -// return DaoUtil.toPageData( -// eventRepository.findErrorEvents( -// tenantId, -// entityId.getId(), -// entityId.getEntityType().name(), -// notNull(pageLink.getStartTime()), -// notNull(pageLink.getEndTime()), -// eventFilter.getServer(), -// eventFilter.getMethod(), -// eventFilter.getErrorStr(), -// DaoUtil.toPageable(pageLink)) -// ); + private PageData findEventByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { + return DaoUtil.toPageData( + errorEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getMethod(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink)) + ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { - return null; -// boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); -// boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); -// return DaoUtil.toPageData( -// eventRepository.findLifeCycleEvents( -// tenantId, -// entityId.getId(), -// entityId.getEntityType().name(), -// notNull(pageLink.getStartTime()), -// notNull(pageLink.getEndTime()), -// eventFilter.getServer(), -// eventFilter.getEvent(), -// statusFilterEnabled, -// statusFilter, -// eventFilter.getErrorStr(), -// DaoUtil.toPageable(pageLink)) -// ); + private PageData findEventByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, TimePageLink pageLink) { + boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); + boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); + return DaoUtil.toPageData( + lcEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getEvent(), + statusFilterEnabled, + statusFilter, + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink)) + ); } private PageData findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { @@ -332,7 +328,7 @@ public class JpaBaseEventDao extends JpaAbstractDao implemen } @Override - public List findLatestEvents(UUID tenantId, EntityId entityId, String eventType, int limit) { + public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return null; // List latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( // tenantId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java new file mode 100644 index 0000000000..f95959ad29 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; + +import java.util.UUID; + +/** + * Created by Valerii Sosliuk on 5/3/2017. + */ +public interface LifecycleEventRepository extends JpaRepository { + + @Query("SELECT e FROM LifecycleEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String eventType, + @Param("statusFilterEnabled") boolean statusFilterEnabled, + @Param("statusFilter") boolean statusFilter, + @Param("error") String error, + Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java new file mode 100644 index 0000000000..2d41501933 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; + +import java.util.UUID; + +/** + * Created by Valerii Sosliuk on 5/3/2017. + */ +public interface StatisticsEventRepository extends JpaRepository { + + @Query("SELECT e FROM StatisticsEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEventsWithoutFilter(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + +} diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index 7e15b419a8..b8fc60ecca 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -21,18 +21,18 @@ -- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts -- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) -CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event - (ts DESC NULLS LAST) - WITH (FILLFACTOR=95); +CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); -COMMENT ON INDEX public.idx_event_ts - IS 'This index helps to delete events by TTL using timestamp'; +CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); -CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event - (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) - WITH (FILLFACTOR=95); +CREATE INDEX IF NOT EXISTS idx_stats_event_main + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_lc_event_main + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_error_event_main + ON error_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); -COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 9494e4bdd6..1ab89c7885 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -324,7 +324,7 @@ CREATE TABLE IF NOT EXISTS device_credentials ( ); CREATE TABLE IF NOT EXISTS rule_node_debug_event ( - id uuid NOT NULL CONSTRAINT rule_node_debug_event_pkey PRIMARY KEY, + id uuid NOT NULL, tenant_id uuid NOT NULL , ts bigint NOT NULL, entity_id uuid NOT NULL, @@ -342,7 +342,7 @@ CREATE TABLE IF NOT EXISTS rule_node_debug_event ( ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( - id uuid NOT NULL CONSTRAINT rule_chain_debug_event_pkey PRIMARY KEY, + id uuid NOT NULL, tenant_id uuid NOT NULL, ts bigint NOT NULL, entity_id uuid NOT NULL, @@ -352,17 +352,17 @@ CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS stats_event ( - id uuid NOT NULL CONSTRAINT stats_event_pkey PRIMARY KEY, + id uuid NOT NULL, tenant_id uuid NOT NULL, ts bigint NOT NULL, entity_id uuid NOT NULL, service_id varchar NOT NULL, e_messages_processed bigint NOT NULL, - e_errors_occured bigint NOT NULL + e_errors_occurred bigint NOT NULL ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS lc_event ( - id uuid NOT NULL CONSTRAINT lc_event_pkey PRIMARY KEY, + id uuid NOT NULL, tenant_id uuid NOT NULL, ts bigint NOT NULL, entity_id uuid NOT NULL, @@ -373,7 +373,7 @@ CREATE TABLE IF NOT EXISTS lc_event ( ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS error_event ( - id uuid NOT NULL CONSTRAINT error_event_pkey PRIMARY KEY, + id uuid NOT NULL, tenant_id uuid NOT NULL, ts bigint NOT NULL, entity_id uuid NOT NULL, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 1276f46621..3c856c2c70 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -194,7 +194,7 @@ public abstract class AbstractServiceTest { } return RuleNodeDebugEvent.builder() .tenantId(tenantId) - .entityId(entityId) + .entityId(entityId.getId()) .serviceId("server A") .data(JacksonUtil.toString(readFromResource("TestJsonData.json"))) .build(); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index ffd15ff5ba..eee70f6624 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -22,6 +22,7 @@ import org.junit.Test; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -80,8 +81,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, - timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink); Assert.assertNotNull(events.getData()); Assert.assertTrue(events.getData().size() == 2); @@ -89,7 +89,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); Assert.assertTrue(events.getData().size() == 1); @@ -111,8 +111,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, - timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink); Assert.assertNotNull(events.getData()); Assert.assertTrue(events.getData().size() == 2); @@ -120,7 +119,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, DataConstants.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); Assert.assertTrue(events.getData().size() == 1); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java index 4381aab18f..16170c10ea 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java @@ -16,34 +16,14 @@ package org.thingsboard.server.dao.sql.event; import com.datastax.oss.driver.api.core.uuid.Uuids; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.junit.After; -import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EventInfo; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EventId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.AbstractJpaDaoTest; import org.thingsboard.server.dao.event.EventDao; -import java.io.IOException; -import java.util.List; import java.util.UUID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.thingsboard.server.common.data.DataConstants.ALARM; -import static org.thingsboard.server.common.data.DataConstants.STATS; - /** * Created by Valerii Sosliuk on 5/5/2017. */ From 219e97ff4a293c4ab300712ab4b3f32ef3eb7d6d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 26 Jul 2022 16:28:55 +0300 Subject: [PATCH 03/13] Implementation of Rule Chain and Rule Node debug events --- .../controller/RuleChainController.java | 1 - .../AbstractRuleEngineControllerTest.java | 1 - .../common/data/event/DebugEventFilter.java | 47 +++++ .../data/event/DebugRuleNodeEventFilter.java | 26 --- .../common/data/event/ErrorEventFilter.java | 2 +- .../server/common/data/event/Event.java | 8 + .../server/common/data/event/EventFilter.java | 6 +- .../data/event/LifeCycleEventFilter.java | 2 +- .../data/event/RuleChainDebugEvent.java | 14 +- ...er.java => RuleChainDebugEventFilter.java} | 17 +- .../common/data/event/RuleNodeDebugEvent.java | 29 ++- ...ent.java => RuleNodeDebugEventFilter.java} | 47 +++-- .../data/event/StatisticsEventFilter.java | 2 +- .../server/dao/event/BaseEventService.java | 6 +- .../server/dao/event/EventDao.java | 12 +- .../server/dao/model/ModelConstants.java | 10 + .../dao/model/sql/AdminSettingsEntity.java | 1 - .../dao/model/sql/DeviceProfileEntity.java | 1 - .../dao/model/sql/ErrorEventEntity.java | 9 - .../server/dao/model/sql/EventEntity.java | 3 - .../dao/model/sql/LifecycleEventEntity.java | 3 - .../model/sql/RuleChainDebugEventEntity.java | 63 ++++++ .../model/sql/RuleNodeDebugEventEntity.java | 109 +++++++++++ .../dao/model/sql/StatisticsEventEntity.java | 8 - .../dao/sql/event/ErrorEventRepository.java | 14 +- .../dao/sql/event/EventInsertRepository.java | 2 - .../server/dao/sql/event/EventRepository.java | 58 +----- .../server/dao/sql/event/JpaBaseEventDao.java | 185 ++++++++---------- .../sql/event/LifecycleEventRepository.java | 12 +- .../event/RuleChainDebugEventRepository.java | 82 ++++++++ .../event/RuleNodeDebugEventRepository.java | 103 ++++++++++ .../sql/event/StatisticsEventRepository.java | 49 ++++- .../sql/schema-entities-idx-psql-addon.sql | 10 +- .../main/resources/sql/schema-entities.sql | 6 +- .../components/event/event-table-config.ts | 4 +- ui-ngx/src/app/shared/models/event.models.ts | 2 +- 36 files changed, 667 insertions(+), 287 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java rename common/data/src/main/java/org/thingsboard/server/common/data/event/{DebugRuleChainEventFilter.java => RuleChainDebugEventFilter.java} (61%) rename common/data/src/main/java/org/thingsboard/server/common/data/event/{DebugEvent.java => RuleNodeDebugEventFilter.java} (59%) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index bcca7965b3..5c79f9eca7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -39,7 +39,6 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.rule.engine.api.ScriptEngine; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.tenant.DebugTbRateLimits; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java index ca0d5a6215..a3edae8ed6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractRuleEngineControllerTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.context.TestPropertySource; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java new file mode 100644 index 0000000000..c895b61c78 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEventFilter.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.event; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.springframework.data.domain.Page; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.StringUtils; + +import java.util.UUID; + +@Data +@ApiModel +public abstract class DebugEventFilter implements EventFilter { + + @ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") + protected String server; + @ApiModelProperty(position = 10, value = "Boolean value to filter the errors", allowableValues = "false, true") + protected boolean isError; + @ApiModelProperty(position = 11, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB") + protected String errorStr; + + public void setIsError(boolean isError) { + this.isError = isError; + } + + @Override + public boolean isNotEmpty() { + return !StringUtils.isEmpty(server) || isError || !StringUtils.isEmpty(errorStr); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java deleted file mode 100644 index ec55c8d1a2..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleNodeEventFilter.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.event; - -import io.swagger.annotations.ApiModel; - -@ApiModel -public class DebugRuleNodeEventFilter extends DebugEvent { - @Override - public EventType getEventType() { - return EventType.DEBUG_RULE_NODE; - } -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java index 9332cd283a..c8f2106764 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEventFilter.java @@ -37,7 +37,7 @@ public class ErrorEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { + public boolean isNotEmpty() { return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(method) || !StringUtils.isEmpty(errorStr); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java index 6e41a3750e..801289791a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.event; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.BaseData; @@ -56,8 +57,15 @@ public abstract class Event extends BaseData { eventInfo.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); eventInfo.setType(getType().getOldName()); eventInfo.setId(id); + eventInfo.setUid(id.toString()); eventInfo.setCreatedTime(createdTime); eventInfo.setBody(OBJECT_MAPPER.createObjectNode().put("server", getServiceId())); return eventInfo; } + + protected static void putNotNull(ObjectNode json, String key, String value) { + if (value != null) { + json.put(key, value); + } + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java index 5d02fbcd13..39502eba3b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventFilter.java @@ -26,8 +26,8 @@ import io.swagger.annotations.ApiModelProperty; include = JsonTypeInfo.As.PROPERTY, property = "eventType") @JsonSubTypes({ - @JsonSubTypes.Type(value = DebugRuleNodeEventFilter.class, name = "DEBUG_RULE_NODE"), - @JsonSubTypes.Type(value = DebugRuleChainEventFilter.class, name = "DEBUG_RULE_CHAIN"), + @JsonSubTypes.Type(value = RuleNodeDebugEventFilter.class, name = "DEBUG_RULE_NODE"), + @JsonSubTypes.Type(value = RuleChainDebugEventFilter.class, name = "DEBUG_RULE_CHAIN"), @JsonSubTypes.Type(value = ErrorEventFilter.class, name = "ERROR"), @JsonSubTypes.Type(value = LifeCycleEventFilter.class, name = "LC_EVENT"), @JsonSubTypes.Type(value = StatisticsEventFilter.class, name = "STATS") @@ -37,6 +37,6 @@ public interface EventFilter { @ApiModelProperty(position = 1, required = true, value = "String value representing the event type", example = "STATS") EventType getEventType(); - boolean hasFilterForJsonBody(); + boolean isNotEmpty(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java index de50f1579f..104939d202 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifeCycleEventFilter.java @@ -39,7 +39,7 @@ public class LifeCycleEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { + public boolean isNotEmpty() { return !StringUtils.isEmpty(server) || !StringUtils.isEmpty(event) || !StringUtils.isEmpty(status) || !StringUtils.isEmpty(errorStr); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java index 52c961c1c8..7de37d1b5b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -22,7 +23,6 @@ import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -40,9 +40,11 @@ public class RuleChainDebugEvent extends Event { this.error = error; } - @Getter @Setter + @Getter + @Setter private String message; - @Getter @Setter + @Getter + @Setter private String error; @Override @@ -52,6 +54,10 @@ public class RuleChainDebugEvent extends Event { @Override public EventInfo toInfo(EntityType entityType) { - return null; + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + putNotNull(json, "message", message); + putNotNull(json, "error", error); + return eventInfo; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java similarity index 61% rename from common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java rename to common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java index 3569e097d0..f40471a35f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugRuleChainEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEventFilter.java @@ -16,11 +16,26 @@ package org.thingsboard.server.common.data.event; import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.StringUtils; +@Data +@EqualsAndHashCode(callSuper = true) @ApiModel -public class DebugRuleChainEventFilter extends DebugEvent { +public class RuleChainDebugEventFilter extends DebugEventFilter { + + @ApiModelProperty(position = 2, value = "String value representing the message") + protected String message; + @Override public EventType getEventType() { return EventType.DEBUG_RULE_CHAIN; } + + @Override + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(message); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java index 891ece5f22..fdca84239d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -62,14 +63,16 @@ public class RuleNodeDebugEvent extends Event { private final String dataType; @Getter private final String relationType; - @Getter @Setter + @Getter + @Setter private String data; - @Getter @Setter + @Getter + @Setter private String metadata; - @Getter @Setter + @Getter + @Setter private String error; - //TODO: rename the enum constant @Override public EventType getType() { return EventType.DEBUG_RULE_NODE; @@ -77,6 +80,22 @@ public class RuleNodeDebugEvent extends Event { @Override public EventInfo toInfo(EntityType entityType) { - return null; + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("type", eventType); + if (eventEntity != null) { + json.put("entityId", eventEntity.getId().toString()).put("entityType", eventEntity.getEntityType().name()); + } + if (msgId != null) { + json.put("msgId", msgId.toString()); + } + putNotNull(json, "msgType", msgType); + putNotNull(json, "dataType", dataType); + putNotNull(json, "relationType", relationType); + putNotNull(json, "data", data); + putNotNull(json, "metadata", metadata); + putNotNull(json, "error", error); + return eventInfo; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java similarity index 59% rename from common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java rename to common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java index 90bbff0bd5..aa44141365 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/DebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEventFilter.java @@ -18,41 +18,40 @@ package org.thingsboard.server.common.data.event; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.StringUtils; @Data +@EqualsAndHashCode(callSuper = true) @ApiModel -public abstract class DebugEvent implements EventFilter { +public class RuleNodeDebugEventFilter extends DebugEventFilter { - @ApiModelProperty(position = 1, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT") + @ApiModelProperty(position = 2, value = "String value representing msg direction type (incoming to entity or outcoming from entity)", allowableValues = "IN, OUT") protected String msgDirectionType; - @ApiModelProperty(position = 2, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") - protected String server; - @ApiModelProperty(position = 3, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity") - protected String dataSearch; - @ApiModelProperty(position = 4, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName") - protected String metadataSearch; - @ApiModelProperty(position = 5, value = "String value representing the entity type", allowableValues = "DEVICE") - protected String entityName; - @ApiModelProperty(position = 6, value = "String value representing the type of message routing", example = "Success") - protected String relationType; - @ApiModelProperty(position = 7, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") + @ApiModelProperty(position = 3, value = "String value representing the entity id in the event body (originator of the message)", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") protected String entityId; - @ApiModelProperty(position = 8, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") + @ApiModelProperty(position = 4, value = "String value representing the entity type", allowableValues = "DEVICE") + protected String entityType; + @ApiModelProperty(position = 5, value = "String value representing the message id in the rule engine", example = "de9d54a0-2b7a-11ec-a3cc-23386423d98f") + protected String msgId; + @ApiModelProperty(position = 6, value = "String value representing the message type", example = "POST_TELEMETRY_REQUEST") protected String msgType; - @ApiModelProperty(position = 9, value = "Boolean value to filter the errors", allowableValues = "false, true") - protected boolean isError; - @ApiModelProperty(position = 10, value = "The case insensitive 'contains' filter based on error message", example = "not present in the DB") - protected String errorStr; + @ApiModelProperty(position = 7, value = "String value representing the type of message routing", example = "Success") + protected String relationType; + @ApiModelProperty(position = 8, value = "The case insensitive 'contains' filter based on data (key and value) for the message.", example = "humidity") + protected String dataSearch; + @ApiModelProperty(position = 9, value = "The case insensitive 'contains' filter based on metadata (key and value) for the message.", example = "deviceName") + protected String metadataSearch; - public void setIsError(boolean isError) { - this.isError = isError; + @Override + public EventType getEventType() { + return EventType.DEBUG_RULE_NODE; } @Override - public boolean hasFilterForJsonBody() { - return !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(server) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch) - || !StringUtils.isEmpty(entityName) || !StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(entityId) || !StringUtils.isEmpty(msgType) || !StringUtils.isEmpty(errorStr) || isError; + public boolean isNotEmpty() { + return super.isNotEmpty() || !StringUtils.isEmpty(msgDirectionType) || !StringUtils.isEmpty(entityId) + || !StringUtils.isEmpty(entityType) || !StringUtils.isEmpty(msgId) || !StringUtils.isEmpty(msgType) || + !StringUtils.isEmpty(relationType) || !StringUtils.isEmpty(dataSearch) || !StringUtils.isEmpty(metadataSearch); } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java index b132ff02da..a48d03dcce 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java @@ -37,7 +37,7 @@ public class StatisticsEventFilter implements EventFilter { } @Override - public boolean hasFilterForJsonBody() { + public boolean isNotEmpty() { return !StringUtils.isEmpty(server) || (messagesProcessed != null && messagesProcessed > 0) || (errorsOccurred != null && errorsOccurred > 0); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 311af9af44..85088667f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -102,7 +102,7 @@ public class BaseEventService implements EventService { @Override public List findLatestEvents(TenantId tenantId, EntityId entityId, EventType eventType, int limit) { - return eventDao.findLatestEvents(tenantId.getId(), entityId.getId(), eventType, limit); + return convert(entityId.getEntityType(), eventDao.findLatestEvents(tenantId.getId(), entityId.getId(), eventType, limit)); } @Override @@ -142,4 +142,8 @@ public class BaseEventService implements EventService { pd.getData().stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()) , pd.getTotalPages(), pd.getTotalElements(), pd.hasNext()); } + + private List convert(EntityType entityType, List list) { + return list == null ? null : list.stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 1878594fb4..718cb41d7c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -41,16 +41,6 @@ public interface EventDao { */ ListenableFuture saveAsync(Event event); - /** - * Find events by tenantId, entityId and pageLink. - * - * @param tenantId the tenantId - * @param entityId the entityId - * @param pageLink the pageLink - * @return the event list - */ - PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink); - /** * Find events by tenantId, entityId, eventType and pageLink. * @@ -73,7 +63,7 @@ public interface EventDao { * @param limit the limit * @return the event list */ - List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit); + List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit); /** * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 37934ed306..171417402d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -388,6 +388,16 @@ public class ModelConstants { public static final String EVENT_ERROR_COLUMN_NAME = "e_error"; public static final String EVENT_SUCCESS_COLUMN_NAME = "e_success"; + public static final String EVENT_ENTITY_ID_COLUMN_NAME = "e_entity_id"; + public static final String EVENT_ENTITY_TYPE_COLUMN_NAME = "e_entity_type"; + public static final String EVENT_MSG_ID_COLUMN_NAME = "e_msg_id"; + public static final String EVENT_MSG_TYPE_COLUMN_NAME = "e_msg_type"; + public static final String EVENT_DATA_TYPE_COLUMN_NAME = "e_data_type"; + public static final String EVENT_RELATION_TYPE_COLUMN_NAME = "e_relation_type"; + public static final String EVENT_DATA_COLUMN_NAME = "e_data"; + public static final String EVENT_METADATA_COLUMN_NAME = "e_metadata"; + public static final String EVENT_MESSAGE_COLUMN_NAME = "e_message"; + public static final String DEBUG_MODE = "debug_mode"; /** diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 4da17d2c34..8e7474f96e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; - import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index f2d1970238..893c9d549c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseSqlEntity; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java index 397824140f..2175c97c94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/ErrorEventEntity.java @@ -18,30 +18,21 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.event.ErrorEvent; -import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; -import java.util.UUID; - import static org.thingsboard.server.dao.model.ModelConstants.ERROR_EVENT_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METHOD_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME; @Data @EqualsAndHashCode(callSuper = true) @Entity -@TypeDef(name = "json", typeClass = JsonStringType.class) @Table(name = ERROR_EVENT_TABLE_NAME) @NoArgsConstructor public class ErrorEventEntity extends EventEntity implements BaseEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java index b0a0135849..0e78d22600 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java @@ -17,11 +17,9 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; import lombok.NoArgsConstructor; -import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Id; @@ -34,7 +32,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PR import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN; @Data -@TypeDef(name = "json", typeClass = JsonStringType.class) @NoArgsConstructor @MappedSuperclass public abstract class EventEntity implements BaseEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java index 0a4380cb63..874cc4ea00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/LifecycleEventEntity.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; @@ -36,7 +34,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAM @Data @EqualsAndHashCode(callSuper = true) @Entity -@TypeDef(name = "json", typeClass = JsonStringType.class) @Table(name = LC_EVENT_TABLE_NAME) @NoArgsConstructor public class LifecycleEventEntity extends EventEntity implements BaseEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java new file mode 100644 index 0000000000..7968521f62 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainDebugEventEntity.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.RULE_CHAIN_DEBUG_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = RULE_CHAIN_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class RuleChainDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_MESSAGE_COLUMN_NAME) + private String message; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public RuleChainDebugEventEntity(RuleChainDebugEvent event) { + super(event); + this.message = event.getMessage(); + this.error = event.getError(); + } + + @Override + public RuleChainDebugEvent toData() { + return RuleChainDebugEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .message(message) + .error(error).build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java new file mode 100644 index 0000000000..6e75515b14 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeDebugEventEntity.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.BaseEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; +import java.util.UUID; + +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_DATA_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_METADATA_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_ID_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MSG_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_RELATION_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; +import static org.thingsboard.server.dao.model.ModelConstants.RULE_NODE_DEBUG_EVENT_TABLE_NAME; + +@Data +@EqualsAndHashCode(callSuper = true) +@Entity +@Table(name = RULE_NODE_DEBUG_EVENT_TABLE_NAME) +@NoArgsConstructor +public class RuleNodeDebugEventEntity extends EventEntity implements BaseEntity { + + @Column(name = EVENT_TYPE_COLUMN_NAME) + private String eventType; + @Column(name = EVENT_ENTITY_ID_COLUMN_NAME) + private UUID eventEntityId; + @Column(name = EVENT_ENTITY_TYPE_COLUMN_NAME) + private String eventEntityType; + @Column(name = EVENT_MSG_ID_COLUMN_NAME) + private UUID msgId; + @Column(name = EVENT_MSG_TYPE_COLUMN_NAME) + private String msgType; + @Column(name = EVENT_DATA_TYPE_COLUMN_NAME) + private String dataType; + @Column(name = EVENT_RELATION_TYPE_COLUMN_NAME) + private String relationType; + @Column(name = EVENT_DATA_COLUMN_NAME) + private String data; + @Column(name = EVENT_METADATA_COLUMN_NAME) + private String metadata; + @Column(name = EVENT_ERROR_COLUMN_NAME) + private String error; + + public RuleNodeDebugEventEntity(RuleNodeDebugEvent event) { + super(event); + this.eventType = event.getEventType(); + if (event.getEventEntity() != null) { + this.eventEntityId = event.getEventEntity().getId(); + this.eventEntityType = event.getEventEntity().getEntityType().name(); + } + this.msgId = event.getMsgId(); + this.msgType = event.getMsgType(); + this.dataType = event.getDataType(); + this.relationType = event.getRelationType(); + this.data = event.getData(); + this.metadata = event.getMetadata(); + this.error = event.getError(); + } + + @Override + public RuleNodeDebugEvent toData() { + var builder = RuleNodeDebugEvent.builder() + .tenantId(TenantId.fromUUID(tenantId)) + .entityId(entityId) + .serviceId(serviceId) + .id(id) + .ts(ts) + .eventType(eventType) + .msgId(msgId) + .msgType(msgType) + .dataType(dataType) + .relationType(relationType) + .data(data) + .metadata(metadata) + .error(error); + if (eventEntityId != null) { + builder.eventEntity(EntityIdFactory.getByTypeAndUuid(eventEntityType, eventEntityId)); + } + return builder.build(); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java index 240e6fdcf7..9e151a1c24 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/StatisticsEventEntity.java @@ -18,29 +18,21 @@ package org.thingsboard.server.dao.model.sql; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; -import org.hibernate.annotations.TypeDef; -import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; -import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERRORS_OCCURRED_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ERROR_COLUMN_NAME; import static org.thingsboard.server.dao.model.ModelConstants.EVENT_MESSAGES_PROCESSED_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_SUCCESS_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_COLUMN_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.LC_EVENT_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.STATS_EVENT_TABLE_NAME; @Data @EqualsAndHashCode(callSuper = true) @Entity -@TypeDef(name = "json", typeClass = JsonStringType.class) @Table(name = STATS_EVENT_TABLE_NAME) @NoArgsConstructor public class StatisticsEventEntity extends EventEntity implements BaseEntity { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java index 721ffe42fe..b4fcfcdcc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java @@ -20,17 +20,23 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; +import java.util.List; import java.util.UUID; -/** - * Created by Valerii Sosliuk on 5/3/2017. - */ -public interface ErrorEventRepository extends JpaRepository { +public interface ErrorEventRepository extends EventRepository, JpaRepository { + + @Override + @Query("SELECT e FROM ErrorEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") + List findLatestEvents(UUID tenantId, UUID entityId, int limit); + + @Override @Query("SELECT e FROM ErrorEventEntity e WHERE " + "e.tenantId = :tenantId " + "AND e.entityId = :entityId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java index 79d7d250e7..85220ddf9f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventInsertRepository.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.dao.sql.event; -import org.jetbrains.annotations.NotNull; -import org.postgresql.core.Oid; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.BatchPreparedStatementSetter; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index 8a967395ab..46e0ff6662 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -21,6 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.dao.model.sql.EventEntity; import java.util.List; @@ -29,58 +31,12 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/3/2017. */ -public interface EventRepository extends JpaRepository { +public interface EventRepository, V extends Event> { + + List findLatestEvents(UUID tenantId, UUID entityId, int limit); + + Page findEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime, Pageable pageable); -// EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, -// EntityType entityType, -// UUID entityId, -// String eventType, -// String eventUid); -// -// EventEntity findByTenantIdAndEntityTypeAndEntityId(UUID tenantId, -// EntityType entityType, -// UUID entityId); -// -// @Query("SELECT e FROM EventEntity e WHERE e.tenantId = :tenantId AND e.entityType = :entityType " + -// "AND e.entityId = :entityId AND e.eventType = :eventType ORDER BY e.eventType DESC, e.id DESC") -// List findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( -// @Param("tenantId") UUID tenantId, -// @Param("entityType") EntityType entityType, -// @Param("entityId") UUID entityId, -// @Param("eventType") String eventType, -// Pageable pageable); -// -// @Query("SELECT e FROM EventEntity e WHERE " + -// "e.tenantId = :tenantId " + -// "AND e.entityType = :entityType AND e.entityId = :entityId " + -// "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + -// "AND (:endTime IS NULL OR e.createdTime <= :endTime) " + -// "AND LOWER(e.eventType) LIKE LOWER(CONCAT('%', :textSearch, '%'))" -// ) -// Page findEventsByTenantIdAndEntityId(@Param("tenantId") UUID tenantId, -// @Param("entityType") EntityType entityType, -// @Param("entityId") UUID entityId, -// @Param("textSearch") String textSearch, -// @Param("startTime") Long startTime, -// @Param("endTime") Long endTime, -// Pageable pageable); -// -// -// -// @Query("SELECT e FROM EventEntity e WHERE " + -// "e.tenantId = :tenantId " + -// "AND e.entityType = :entityType AND e.entityId = :entityId " + -// "AND e.eventType = :eventType " + -// "AND (:startTime IS NULL OR e.createdTime >= :startTime) " + -// "AND (:endTime IS NULL OR e.createdTime <= :endTime)" -// ) -// Page findEventsByTenantIdAndEntityIdAndEventType(@Param("tenantId") UUID tenantId, -// @Param("entityType") EntityType entityType, -// @Param("entityId") UUID entityId, -// @Param("eventType") String eventType, -// @Param("startTime") Long startTime, -// @Param("endTime") Long endTime, -// Pageable pageable); // // @Query(nativeQuery = true, // value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 0717fa6896..26117637d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -23,25 +23,22 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.event.DebugEvent; -import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.RuleChainDebugEventFilter; +import org.thingsboard.server.common.data.event.RuleNodeDebugEventFilter; import org.thingsboard.server.common.data.event.ErrorEventFilter; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.event.LifeCycleEventFilter; -import org.thingsboard.server.common.data.event.LifecycleEvent; -import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.common.data.event.StatisticsEventFilter; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.event.EventDao; +import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; @@ -51,6 +48,7 @@ import org.thingsboard.server.dao.timeseries.SqlPartition; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Comparator; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -74,9 +72,6 @@ public class JpaBaseEventDao implements EventDao { @Autowired private SqlPartitioningRepository partitioningRepository; - @Autowired - private EventRepository eventRepository; - @Autowired private LifecycleEventRepository lcEventRepository; @@ -92,6 +87,12 @@ public class JpaBaseEventDao implements EventDao { @Autowired private EventCleanupRepository eventCleanupRepository; + @Autowired + private RuleNodeDebugEventRepository ruleNodeDebugEventRepository; + + @Autowired + private RuleChainDebugEventRepository ruleChainDebugEventRepository; + @Autowired ScheduledLogExecutorComponent logExecutor; @@ -115,6 +116,8 @@ public class JpaBaseEventDao implements EventDao { private TbSqlBlockingQueueWrapper queue; + private final Map> repositories = new ConcurrentHashMap<>(); + @PostConstruct private void init() { for (EventType eventType : EventType.values()) { @@ -131,6 +134,11 @@ public class JpaBaseEventDao implements EventDao { Function hashcodeFunction = entity -> Objects.hash(super.hashCode(), entity.getTenantId(), entity.getEntityId()); queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, batchThreads, statsFactory); queue.init(logExecutor, v -> eventInsertRepository.save(v), Comparator.comparing(Event::getCreatedTime)); + repositories.put(EventType.LC_EVENT, lcEventRepository); + repositories.put(EventType.STATS, statsEventRepository); + repositories.put(EventType.ERROR, errorEventRepository); + repositories.put(EventType.DEBUG_RULE_NODE, ruleNodeDebugEventRepository); + repositories.put(EventType.DEBUG_RULE_CHAIN, ruleChainDebugEventRepository); } @PreDestroy @@ -190,58 +198,19 @@ public class JpaBaseEventDao implements EventDao { } } - @Override - public PageData findEvents(UUID tenantId, EntityId entityId, TimePageLink pageLink) { - return null; -// return DaoUtil.toPageData( -// eventRepository -// .findEventsByTenantIdAndEntityId( -// tenantId, -// entityId.getEntityType(), -// entityId.getId(), -// Objects.toString(pageLink.getTextSearch(), ""), -// pageLink.getStartTime(), -// pageLink.getEndTime(), -// DaoUtil.toPageable(pageLink))); - } - @Override public PageData findEvents(UUID tenantId, UUID entityId, EventType eventType, TimePageLink pageLink) { - switch (eventType) { - case LC_EVENT: - return findLcEventsWithoutFilter(tenantId, entityId, pageLink); - case STATS: - return findStatsEventsWithoutFilter(tenantId, entityId, pageLink); - case ERROR: - return findErrorEventsWithoutFilter(tenantId, entityId, pageLink); - default: - throw new RuntimeException("Event type: " + eventType + " is not supported!"); - } + return DaoUtil.toPageData(getEventRepository(eventType).findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); } - private PageData findLcEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { - return DaoUtil.toPageData( - lcEventRepository.findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); - } - - private PageData findStatsEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { - return DaoUtil.toPageData( - statsEventRepository.findEventsWithoutFilter(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); - } - - private PageData findErrorEventsWithoutFilter(UUID tenantId, UUID entityId, TimePageLink pageLink) { - return DaoUtil.toPageData( - errorEventRepository.findEvents(tenantId, entityId, pageLink.getStartTime(), pageLink.getEndTime(), DaoUtil.toPageable(pageLink))); - } - - @Override public PageData findEventByFilter(UUID tenantId, UUID entityId, EventFilter eventFilter, TimePageLink pageLink) { - if (eventFilter.hasFilterForJsonBody()) { + if (eventFilter.isNotEmpty()) { switch (eventFilter.getEventType()) { case DEBUG_RULE_NODE: + return findEventByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, pageLink); case DEBUG_RULE_CHAIN: - return findEventByFilter(tenantId, entityId, (DebugEvent) eventFilter, pageLink); + return findEventByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, pageLink); case LC_EVENT: return findEventByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, pageLink); case ERROR: @@ -256,27 +225,41 @@ public class JpaBaseEventDao implements EventDao { } } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, DebugEvent eventFilter, TimePageLink pageLink) { - return null; -// return DaoUtil.toPageData( -// eventRepository.findDebugRuleNodeEvents( -// tenantId, -// entityId.getId(), -// entityId.getEntityType().name(), -// eventFilter.getEventType().name(), -// notNull(pageLink.getStartTime()), -// notNull(pageLink.getEndTime()), -// eventFilter.getMsgDirectionType(), -// eventFilter.getServer(), -// eventFilter.getEntityName(), -// eventFilter.getRelationType(), -// eventFilter.getEntityId(), -// eventFilter.getMsgType(), -// eventFilter.isError(), -// eventFilter.getErrorStr(), -// eventFilter.getDataSearch(), -// eventFilter.getMetadataSearch(), -// DaoUtil.toPageable(pageLink))); + private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, TimePageLink pageLink) { + return DaoUtil.toPageData( + ruleChainDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getMessage(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink))); + } + + private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, TimePageLink pageLink) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + return DaoUtil.toPageData( + ruleNodeDebugEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getMsgDirectionType(), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getRelationType(), + eventFilter.getDataSearch(), + eventFilter.getMetadataSearch(), + eventFilter.isError(), + eventFilter.getErrorStr(), + DaoUtil.toPageable(pageLink))); } private PageData findEventByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, TimePageLink pageLink) { @@ -311,32 +294,23 @@ public class JpaBaseEventDao implements EventDao { ); } - private PageData findEventByFilter(UUID tenantId, EntityId entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { - return null; -// return DaoUtil.toPageData( -// eventRepository.findStatisticsEvents( -// tenantId, -// entityId.getId(), -// entityId.getEntityType().name(), -// notNull(pageLink.getStartTime()), -// notNull(pageLink.getEndTime()), -// eventFilter.getServer(), -// notNull(eventFilter.getMessagesProcessed()), -// notNull(eventFilter.getErrorsOccurred()), -// DaoUtil.toPageable(pageLink)) -// ); + private PageData findEventByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, TimePageLink pageLink) { + return DaoUtil.toPageData( + statsEventRepository.findEvents( + tenantId, + entityId, + pageLink.getStartTime(), + pageLink.getEndTime(), + eventFilter.getServer(), + eventFilter.getMessagesProcessed(), + eventFilter.getErrorsOccurred(), + DaoUtil.toPageable(pageLink)) + ); } @Override - public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { - return null; -// List latest = eventRepository.findLatestByTenantIdAndEntityTypeAndEntityIdAndEventType( -// tenantId, -// entityId.getEntityType(), -// entityId.getId(), -// eventType, -// PageRequest.of(0, limit)); -// return DaoUtil.convertDataList(latest); + public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { + return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); } @Override @@ -345,12 +319,23 @@ public class JpaBaseEventDao implements EventDao { eventCleanupRepository.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); } - private long notNull(Long value) { - return value != null ? value : 0; + private void parseUUID(String src, String paramName) { + if (!StringUtils.isEmpty(src)) { + try { + UUID.fromString(src); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Failed to convert " + paramName + " to UUID!"); + } + } } - private int notNull(Integer value) { - return value != null ? value : 0; + private EventRepository, ?> getEventRepository(EventType eventType) { + var repository = repositories.get(eventType); + if (repository == null) { + throw new RuntimeException("Event type: " + eventType + " is not supported!"); + } + return repository; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java index f95959ad29..298e57a1b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -20,14 +20,18 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import java.util.List; import java.util.UUID; -/** - * Created by Valerii Sosliuk on 5/3/2017. - */ -public interface LifecycleEventRepository extends JpaRepository { +public interface LifecycleEventRepository extends EventRepository, JpaRepository { + + @Override + @Query("SELECT e FROM LifecycleEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") + List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query("SELECT e FROM LifecycleEventEntity e WHERE " + "e.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java new file mode 100644 index 0000000000..32d27765a5 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.event.RuleChainDebugEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; +import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity; + +import java.util.List; +import java.util.UUID; + + +public interface RuleChainDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query("SELECT e FROM RuleChainDebugEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") + List findLatestEvents(UUID tenantId, UUID entityId, int limit); + + @Override + @Query("SELECT e FROM RuleChainDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("message") String message, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java new file mode 100644 index 0000000000..6238acd5ac --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java @@ -0,0 +1,103 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.event.ErrorEvent; +import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.dao.model.sql.ErrorEventEntity; +import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity; + +import java.util.List; +import java.util.UUID; + + +public interface RuleNodeDebugEventRepository extends EventRepository, JpaRepository { + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") + List findLatestEvents(UUID tenantId, UUID entityId, int limit); + + @Override + @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + , + countQuery = "SELECT count(*) FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String type, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("relationType") String relationType, + @Param("data") String data, + @Param("metadata") String metadata, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java index 2d41501933..5eee96efbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java @@ -21,15 +21,16 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.event.StatisticsEvent; -import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; +import java.util.List; import java.util.UUID; -/** - * Created by Valerii Sosliuk on 5/3/2017. - */ -public interface StatisticsEventRepository extends JpaRepository { +public interface StatisticsEventRepository extends EventRepository, JpaRepository { + + @Override + @Query("SELECT e FROM LifecycleEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") + List findLatestEvents(UUID tenantId, UUID entityId, int limit); @Query("SELECT e FROM StatisticsEventEntity e WHERE " + "e.tenantId = :tenantId " + @@ -37,10 +38,38 @@ public interface StatisticsEventRepository extends JpaRepository= :startTime) " + "AND (:endTime IS NULL OR e.ts <= :endTime)" ) - Page findEventsWithoutFilter(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); + + @Query(nativeQuery = true, + value = "SELECT * FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:messagesProcessed IS NULL OR e.e_messages_processed >= :messagesProcessed) " + + "AND (:errorsOccurred IS NULL OR e.e_errors_occurred >= :errorsOccurred)" + , + countQuery = "SELECT count(*) FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:messagesProcessed IS NULL OR e.e_messages_processed >= :messagesProcessed) " + + "AND (:errorsOccurred IS NULL OR e.e_errors_occurred >= :errorsOccurred)" + ) + Page findEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("messagesProcessed") Integer messagesProcessed, + @Param("errorsOccurred") Integer errorsOccurred, + Pageable pageable); } diff --git a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql index b8fc60ecca..d78257ad8b 100644 --- a/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql +++ b/dao/src/main/resources/sql/schema-entities-idx-psql-addon.sql @@ -22,17 +22,17 @@ -- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main - ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main - ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); CREATE INDEX IF NOT EXISTS idx_stats_event_main - ON stats_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); CREATE INDEX IF NOT EXISTS idx_lc_event_main - ON lc_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); CREATE INDEX IF NOT EXISTS idx_error_event_main - ON error_event (tenant_id ASC, entity_id ASC, ts DESC) WITH (FILLFACTOR=95); + ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 1ab89c7885..22f36a2bfb 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -336,9 +336,9 @@ CREATE TABLE IF NOT EXISTS rule_node_debug_event ( e_msg_type varchar, e_data_type varchar, e_relation_type varchar, - e_data varchar NOT NULL, - e_metadata varchar NOT NULL, - e_error varchar NOT NULL + e_data varchar, + e_metadata varchar, + e_error varchar ) PARTITION BY RANGE (ts); CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 0aebf12bd5..4d6e43a34a 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -234,8 +234,8 @@ export class EventTableConfig extends EntityTableConfig { }), false, key => ({ padding: '0 12px 0 0' })), - new EntityTableColumn('entityName', 'event.entity-type', '100px', - (entity) => entity.body.entityName, entity => ({ + new EntityTableColumn('entityType', 'event.entity-type', '100px', + (entity) => entity.body.entityType, entity => ({ padding: '0 12px 0 0', }), false, key => ({ padding: '0 12px 0 0' diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index 220e7ce3b2..7ed4203dce 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -65,7 +65,7 @@ export interface StatsEventBody extends BaseEventBody { export interface DebugRuleNodeEventBody extends BaseEventBody { type: string; entityId: string; - entityName: string; + entityType: string; msgId: string; msgType: string; relationType: string; From d8341ebdf7ef69e3ee2e1fa9c4fc16ae43548992 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 26 Jul 2022 17:38:22 +0300 Subject: [PATCH 04/13] UI: Changed event table for new "debug rule chain event" data model --- .../components/event/event-table-config.ts | 33 +++++++++++++++++-- ui-ngx/src/app/shared/models/event.models.ts | 7 +++- .../assets/locale/locale.constant-en_US.json | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 4d6e43a34a..0cb93b582e 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -225,7 +225,6 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_RULE_NODE: - case DebugEventType.DEBUG_RULE_CHAIN: this.columns[0].width = '100px'; this.columns.push( new EntityTableColumn('type', 'event.type', '40px', @@ -289,6 +288,29 @@ export class EventTableConfig extends EntityTableConfig { '40px') ); break; + case DebugEventType.DEBUG_RULE_CHAIN: + this.columns[0].width = '100px'; + this.columns.push( + new EntityActionTableColumn('message', 'event.message', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.message ? entity.body.message.length > 0 : false, + onAction: ($event, entity) => this.showContent($event, entity.body.message, + 'event.message') + }, + '40px'), + new EntityActionTableColumn('error', 'event.error', + { + name: this.translate.instant('action.view'), + icon: 'more_horiz', + isEnabled: (entity) => entity.body.error && entity.body.error.length > 0, + onAction: ($event, entity) => this.showContent($event, entity.body.error, + 'event.error') + }, + '40px') + ); + break; } if (updateTableColumns) { this.getTable().columnsUpdated(true); @@ -339,11 +361,10 @@ export class EventTableConfig extends EntityTableConfig { ); break; case DebugEventType.DEBUG_RULE_NODE: - case DebugEventType.DEBUG_RULE_CHAIN: this.filterColumns.push( {key: 'msgDirectionType', title: 'event.type'}, {key: 'entityId', title: 'event.entity-id'}, - {key: 'entityName', title: 'event.entity-type'}, + {key: 'entityType', title: 'event.entity-type'}, {key: 'msgType', title: 'event.message-type'}, {key: 'relationType', title: 'event.relation-type'}, {key: 'dataSearch', title: 'event.data'}, @@ -352,6 +373,12 @@ export class EventTableConfig extends EntityTableConfig { {key: 'errorStr', title: 'event.error'} ); break; + case DebugEventType.DEBUG_RULE_CHAIN: + this.filterColumns.push( + {key: 'isError', title: 'event.error'}, + {key: 'errorStr', title: 'event.error'} + ); + break; } } diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index 7ed4203dce..89e93d4fa1 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -75,7 +75,12 @@ export interface DebugRuleNodeEventBody extends BaseEventBody { error: string; } -export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody; +export interface DebugRuleChainEventBody extends BaseEventBody { + message: string; + error?: string; +} + +export type EventBody = ErrorEventBody & LcEventEventBody & StatsEventBody & DebugRuleNodeEventBody & DebugRuleChainEventBody; export interface Event extends BaseData { tenantId: TenantId; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index fd1bc2d688..0ea7d08742 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2012,6 +2012,7 @@ "body": "Body", "method": "Method", "type": "Type", + "message": "Message", "message-id": "Message Id", "message-type": "Message Type", "data-type": "Data Type", From 68430e415f72eea541d5d34eacfb51f2346e12bb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 26 Jul 2022 17:58:19 +0300 Subject: [PATCH 05/13] UI: Add news event filter settings --- .../src/app/modules/home/components/event/event-table-config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 0cb93b582e..c094e3696c 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -365,6 +365,7 @@ export class EventTableConfig extends EntityTableConfig { {key: 'msgDirectionType', title: 'event.type'}, {key: 'entityId', title: 'event.entity-id'}, {key: 'entityType', title: 'event.entity-type'}, + {key: 'msgId', title: 'event.message-id'}, {key: 'msgType', title: 'event.message-type'}, {key: 'relationType', title: 'event.relation-type'}, {key: 'dataSearch', title: 'event.data'}, @@ -375,6 +376,7 @@ export class EventTableConfig extends EntityTableConfig { break; case DebugEventType.DEBUG_RULE_CHAIN: this.filterColumns.push( + {key: 'message', title: 'event.message'}, {key: 'isError', title: 'event.error'}, {key: 'errorStr', title: 'event.error'} ); From ab0710c13e9a6a346a0278f754644a3f8a0449ad Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 26 Jul 2022 19:15:25 +0300 Subject: [PATCH 06/13] Cleanup event tables by partitions --- .../service/ttl/EventsCleanUpService.java | 23 +--- .../src/main/resources/thingsboard.yml | 5 +- .../server/dao/event/EventService.java | 2 +- .../server/common/data/event/EventType.java | 13 ++- .../server/dao/event/BaseEventService.java | 4 +- .../server/dao/event/EventDao.java | 8 +- .../dao/sql/event/EventCleanupRepository.java | 2 +- .../server/dao/sql/event/JpaBaseEventDao.java | 23 ++-- .../sql/event/SqlEventCleanupRepository.java | 104 +++++++++++++++--- .../service/event/BaseEventServiceTest.java | 4 +- .../test/resources/cassandra-test.properties | 2 - 11 files changed, 132 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index 1fd3892f39..08ee57119c 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -56,26 +56,9 @@ public class EventsCleanUpService extends AbstractCleanUpService { public void cleanUp() { if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { long ts = System.currentTimeMillis(); - long regularEventStartTs; - long regularEventEndTs; - long debugEventStartTs; - long debugEventEndTs; - - if (ttlInSec > 0) { - regularEventEndTs = ts - TimeUnit.SECONDS.toMillis(ttlInSec); - regularEventStartTs = regularEventEndTs - 2 * executionIntervalInMs; - } else { - regularEventStartTs = regularEventEndTs = 0; - } - - if (debugTtlInSec > 0) { - debugEventEndTs = ts - TimeUnit.SECONDS.toMillis(debugTtlInSec); - debugEventStartTs = debugEventEndTs - 2 * executionIntervalInMs; - } else { - debugEventStartTs = debugEventEndTs = 0; - } - - eventService.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + long regularEventExpTs = ttlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(ttlInSec) : 0; + long debugEventExpTs = debugTtlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(debugTtlInSec) : 0; + eventService.cleanupEvents(regularEventExpTs, debugEventExpTs); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 17b996934e..5015cc2175 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -221,9 +221,6 @@ cassandra: ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}" ts_key_value_ttl: "${TS_KV_TTL:0}" - events_ttl: "${TS_EVENTS_TTL:0}" - # Specify TTL of debug log in seconds. The current value corresponds to one week - debug_events_ttl: "${DEBUG_EVENTS_TTL:604800}" buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}" concurrent_limit: "${CASSANDRA_QUERY_CONCURRENT_LIMIT:1000}" permit_max_wait_time: "${PERMIT_MAX_WAIT_TIME:120000}" @@ -287,7 +284,7 @@ sql: ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:3600000}" # Number of milliseconds (max random initial delay and fixed period). events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week edge_events: diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index b5f4c98e25..2b89959b56 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -42,6 +42,6 @@ public interface EventService { void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime); - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void cleanupEvents(long regularEventExpTs, long debugEventExpTs); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java index 5fa6fb7370..a7909d28b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/EventType.java @@ -18,16 +18,27 @@ package org.thingsboard.server.common.data.event; import lombok.Getter; public enum EventType { - ERROR("error_event", "ERROR"), LC_EVENT("lc_event", "LC_EVENT"), STATS("stats_event", "STATS"), DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE"), DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN"); + ERROR("error_event", "ERROR"), + LC_EVENT("lc_event", "LC_EVENT"), + STATS("stats_event", "STATS"), + DEBUG_RULE_NODE("rule_node_debug_event", "DEBUG_RULE_NODE", true), + DEBUG_RULE_CHAIN("rule_chain_debug_event", "DEBUG_RULE_CHAIN", true); @Getter private final String table; @Getter private final String oldName; + @Getter + private final boolean debug; EventType(String table, String oldName) { + this(table, oldName, false); + } + + EventType(String table, String oldName, boolean debug) { this.table = table; this.oldName = oldName; + this.debug = debug; } } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 85088667f1..302e3f0fe8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -133,8 +133,8 @@ public class BaseEventService implements EventService { } @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { - eventDao.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs) { + eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs); } private PageData convert(EntityType entityType, PageData pd) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 718cb41d7c..b7ca1282c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -67,10 +67,8 @@ public interface EventDao { /** * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. - * @param regularEventStartTs the start time of the interval to use to delete non debug events - * @param regularEventEndTs the end time of the interval to use to delete non debug events - * @param debugEventStartTs the start time of the interval to use to delete debug events - * @param debugEventEndTs the end time of the interval to use to delete debug events + * @param regularEventExpTs the expiration time of the regular events + * @param debugEventExpTs the expiration time of the debug events */ - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void cleanupEvents(long regularEventExpTs, long debugEventExpTs); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java index 757d69f794..8631db46ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java @@ -17,6 +17,6 @@ package org.thingsboard.server.dao.sql.event; public interface EventCleanupRepository { - void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs); + void cleanupEvents(long eventExpTime, boolean debug); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 26117637d5..0a39ab1be0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -65,7 +65,9 @@ import java.util.function.Function; @Component public class JpaBaseEventDao implements EventDao { - private static final long PARTITION_DURATION = TimeUnit.HOURS.toMillis(1); + public static final long REGULAR_PARTITION_DURATION = TimeUnit.DAYS.toMillis(1); + public static final long DEBUG_PARTITION_DURATION = TimeUnit.HOURS.toMillis(1); + private final Map> partitionsByEventType = new ConcurrentHashMap<>(); private static final ReentrantLock partitionCreationLock = new ReentrantLock(); @@ -169,10 +171,11 @@ public class JpaBaseEventDao implements EventDao { private void savePartitionIfNotExist(Event event) { var partitionsMap = partitionsByEventType.get(event.getType()); - long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % PARTITION_DURATION); + var partitionDuration = event.getType().isDebug() ? DEBUG_PARTITION_DURATION : REGULAR_PARTITION_DURATION; + long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % partitionDuration); if (partitionsMap.get(partitionStartTs) == null) { - long partitionEndTs = partitionStartTs + PARTITION_DURATION; - savePartition(partitionsMap, new SqlPartition(event.getType().getTable(), partitionStartTs, partitionEndTs, Long.toString(partitionStartTs))); + savePartition(partitionsMap, new SqlPartition(event.getType().getTable(), partitionStartTs, + partitionStartTs + partitionDuration, Long.toString(partitionStartTs))); } } @@ -314,9 +317,15 @@ public class JpaBaseEventDao implements EventDao { } @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { - log.info("Going to cleanup old events. Interval for regular events: [{}:{}], for debug events: [{}:{}]", regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); - eventCleanupRepository.cleanupEvents(regularEventStartTs, regularEventEndTs, debugEventStartTs, debugEventEndTs); + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs) { + if (regularEventExpTs > 0) { + log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs); + eventCleanupRepository.cleanupEvents(regularEventExpTs, false); + } + if (debugEventExpTs > 0) { + log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs); + eventCleanupRepository.cleanupEvents(debugEventExpTs, true); + } } private void parseUUID(String src, String paramName) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index 6a17f7c740..54b4565097 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -16,38 +16,116 @@ package org.thingsboard.server.dao.sql.event; import lombok.extern.slf4j.Slf4j; +import org.postgresql.util.PSQLException; import org.springframework.stereotype.Repository; +import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.List; + +import static org.thingsboard.server.dao.sql.event.JpaBaseEventDao.DEBUG_PARTITION_DURATION; +import static org.thingsboard.server.dao.sql.event.JpaBaseEventDao.REGULAR_PARTITION_DURATION; @Slf4j @Repository public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorService implements EventCleanupRepository { + private static final String SELECT_PARTITIONS_STMT = "SELECT tablename from pg_tables WHERE schemaname = 'public' and tablename like concat(?, '_%')"; + private static final int PSQL_VERSION_14 = 140000; + + private volatile Integer currentServerVersion; + @Override - public void cleanupEvents(long regularEventStartTs, long regularEventEndTs, long debugEventStartTs, long debugEventEndTs) { + public void cleanupEvents(long eventExpTime, boolean debug) { + for (EventType eventType : EventType.values()) { + if (eventType.isDebug() == debug) { + cleanupEvents(eventType, eventExpTime); + } + } + } + + private void cleanupEvents(EventType eventType, long eventExpTime) { + var partitionDuration = eventType.isDebug() ? DEBUG_PARTITION_DURATION : REGULAR_PARTITION_DURATION; + List partitions = fetchPartitions(eventType); + for (var partitionTs : partitions) { + var partitionEndTs = partitionTs + partitionDuration; + if (partitionEndTs < eventExpTime) { + log.info("[{}] Detaching expired partition: [{}-{}]", eventType, partitionTs, partitionEndTs); + if (detachAndDropPartition(eventType, partitionTs)) { + log.info("[{}] Detached expired partition: {}", eventType, partitionTs); + } + } else { + log.debug("[{}] Skip valid partition: {}", eventType, partitionTs); + } + } + } + + private List fetchPartitions(EventType eventType) { + List partitions = new ArrayList<>(); try (Connection connection = dataSource.getConnection(); - PreparedStatement stmt = connection.prepareStatement("call cleanup_events_by_ttl(?,?,?,?,?)")) { - stmt.setLong(1, regularEventStartTs); - stmt.setLong(2, regularEventEndTs); - stmt.setLong(3, debugEventStartTs); - stmt.setLong(4, debugEventEndTs); - stmt.setLong(5, 0); - stmt.setQueryTimeout((int) TimeUnit.HOURS.toSeconds(1)); + PreparedStatement stmt = connection.prepareStatement(SELECT_PARTITIONS_STMT)) { + stmt.setString(1, eventType.getTable()); stmt.execute(); - printWarnings(stmt); - try (ResultSet resultSet = stmt.getResultSet()){ - resultSet.next(); - log.info("Total events removed by TTL: [{}]", resultSet.getLong(1)); + try (ResultSet resultSet = stmt.getResultSet()) { + while (resultSet.next()) { + String partitionTableName = resultSet.getString(1); + String partitionTsStr = partitionTableName.substring(eventType.getTable().length() + 1); + try { + partitions.add(Long.parseLong(partitionTsStr)); + } catch (NumberFormatException nfe) { + log.warn("Failed to parse table name: {}", partitionTableName); + } + } } } catch (SQLException e) { log.error("SQLException occurred during events TTL task execution ", e); } + return partitions; + } + + private boolean detachAndDropPartition(EventType eventType, long partitionTs) { + String tablePartition = eventType.getTable() + "_" + partitionTs; + String detachPsqlStmtStr = "ALTER TABLE " + eventType.getTable() + " DETACH PARTITION " + tablePartition; + if (getCurrentServerVersion() >= PSQL_VERSION_14) { + detachPsqlStmtStr += " CONCURRENTLY"; + } + + String dropStmtStr = "DROP TABLE " + tablePartition; + try (Connection connection = dataSource.getConnection(); + PreparedStatement detachStmt = connection.prepareStatement(detachPsqlStmtStr); + PreparedStatement dropStmt = connection.prepareStatement(dropStmtStr)) { + detachStmt.execute(); + dropStmt.execute(); + return true; + } catch (SQLException e) { + log.error("[{}] SQLException occurred during detach and drop of the partition: {}", eventType, partitionTs, e); + } + return false; + } + + private synchronized int getCurrentServerVersion() { + if (currentServerVersion == null) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement versionStmt = connection.prepareStatement("SELECT current_setting('server_version_num')")) { + versionStmt.execute(); + try (ResultSet resultSet = versionStmt.getResultSet()) { + while (resultSet.next()) { + currentServerVersion = resultSet.getInt(1); + } + } + } catch (SQLException e) { + log.warn("SQLException occurred during fetch of the server version", e); + } + if (currentServerVersion == null) { + currentServerVersion = 0; + } + } + return currentServerVersion; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index eee70f6624..4bb315e5dc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -96,7 +96,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1); } @Test @@ -126,7 +126,7 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1); } private EventInfo saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { diff --git a/dao/src/test/resources/cassandra-test.properties b/dao/src/test/resources/cassandra-test.properties index 4b3ea0a74d..5765153a6f 100644 --- a/dao/src/test/resources/cassandra-test.properties +++ b/dao/src/test/resources/cassandra-test.properties @@ -58,8 +58,6 @@ cassandra.query.ts_key_value_partitions_max_cache_size=100000 cassandra.query.ts_key_value_ttl=0 -cassandra.query.debug_events_ttl=604800 - cassandra.query.max_limit_per_request=1000 cassandra.query.buffer_size=100000 cassandra.query.concurrent_limit=1000 From 3af07efd7f04502986e7e949ecdb2b7d6a822cdc Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 27 Jul 2022 15:30:08 +0300 Subject: [PATCH 07/13] Support for bulk delete of events --- .../src/main/resources/thingsboard.yml | 8 +- .../data/event/StatisticsEventFilter.java | 14 +- .../server/dao/event/BaseEventService.java | 18 +-- .../server/dao/event/EventDao.java | 23 +++ .../dao/sql/event/ErrorEventRepository.java | 34 +++++ .../event/EventPartitionConfiguration.java | 45 ++++++ .../server/dao/sql/event/EventRepository.java | 129 +---------------- .../server/dao/sql/event/JpaBaseEventDao.java | 132 ++++++++++++++++-- .../sql/event/LifecycleEventRepository.java | 37 +++++ .../event/RuleChainDebugEventRepository.java | 59 ++++++-- .../event/RuleNodeDebugEventRepository.java | 49 +++++++ .../sql/event/SqlEventCleanupRepository.java | 9 +- .../sql/event/StatisticsEventRepository.java | 57 +++++++- 13 files changed, 435 insertions(+), 179 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5015cc2175..1c50fb9ced 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -259,6 +259,8 @@ sql: batch_max_delay: "${SQL_EVENTS_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_EVENTS_BATCH_STATS_PRINT_MS:10000}" batch_threads: "${SQL_EVENTS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution + partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:168}" # Number of hours to partition the events. The current value corresponds to one week. + debug_partition_size: "${SQL_EVENTS_REGULAR_PARTITION_SIZE_HOURS:1}" # Number of hours to partition the debug events. The current value corresponds to one hour. edge_events: batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" @@ -285,8 +287,10 @@ sql: events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:3600000}" # Number of milliseconds (max random initial delay and fixed period). - events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds - debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week + # Number of seconds. TTL is disabled by default. Accuracy of the cleanup depends on the sql.events.partition_size parameter. + events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" + # Number of seconds. The current value corresponds to one week. Accuracy of the cleanup depends on the sql.events.debug_partition_size parameter. + debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" edge_events: enabled: "${SQL_TTL_EDGE_EVENTS_ENABLED:true}" execution_interval_ms: "${SQL_TTL_EDGE_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java index a48d03dcce..598d750c0e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEventFilter.java @@ -27,9 +27,13 @@ public class StatisticsEventFilter implements EventFilter { @ApiModelProperty(position = 1, value = "String value representing the server name, identifier or ip address where the platform is running", example = "ip-172-31-24-152") protected String server; @ApiModelProperty(position = 2, value = "The minimum number of successfully processed messages", example = "25") - protected Integer messagesProcessed; - @ApiModelProperty(position = 3, value = "The minimum number of errors occurred during messages processing", example = "30") - protected Integer errorsOccurred; + protected Integer minMessagesProcessed; + @ApiModelProperty(position = 3, value = "The maximum number of successfully processed messages", example = "250") + protected Integer maxMessagesProcessed; + @ApiModelProperty(position = 4, value = "The minimum number of errors occurred during messages processing", example = "30") + protected Integer minErrorsOccurred; + @ApiModelProperty(position = 5, value = "The maximum number of errors occurred during messages processing", example = "300") + protected Integer maxErrorsOccurred; @Override public EventType getEventType() { @@ -38,6 +42,8 @@ public class StatisticsEventFilter implements EventFilter { @Override public boolean isNotEmpty() { - return !StringUtils.isEmpty(server) || (messagesProcessed != null && messagesProcessed > 0) || (errorsOccurred != null && errorsOccurred > 0); + return !StringUtils.isEmpty(server) + || (minMessagesProcessed != null && minMessagesProcessed > 0) || (minErrorsOccurred != null && minErrorsOccurred > 0) + || (maxMessagesProcessed != null && maxMessagesProcessed > 0) || (maxErrorsOccurred != null && maxErrorsOccurred > 0); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 302e3f0fe8..fad7f6b310 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -117,19 +117,11 @@ public class BaseEventService implements EventService { @Override public void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime) { -// TimePageLink eventsPageLink = new TimePageLink(1000, 0, null, null, startTime, endTime); -// PageData eventsPageData; -// do { -// if (eventFilter == null) { -// eventsPageData = findEvents(tenantId, entityId, eventsPageLink); -// } else { -// eventsPageData = findEventsByFilter(tenantId, entityId, eventFilter, eventsPageLink); -// } -// -// eventDao.removeAllByIds(eventsPageData.getData().stream() -// .map(IdBased::getUuidId) -// .collect(Collectors.toList())); -// } while (eventsPageData.hasNext()); + if (eventFilter == null) { + eventDao.removeEvents(tenantId.getId(), entityId.getId(), startTime, endTime); + } else { + eventDao.removeEvents(tenantId.getId(), entityId.getId(), eventFilter, startTime, endTime); + } } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index b7ca1282c7..e09ea59872 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; @@ -71,4 +72,26 @@ public interface EventDao { * @param debugEventExpTs the expiration time of the debug events */ void cleanupEvents(long regularEventExpTs, long debugEventExpTs); + + /** + * Removes all events for the specified entity and time interval + * + * @param tenantId + * @param entityId + * @param startTime + * @param endTime + */ + void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime); + + /** + * + * Removes all events for the specified entity, event filter and time interval + * + * @param tenantId + * @param entityId + * @param eventFilter + * @param startTime + * @param endTime + */ + void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java index b4fcfcdcc7..c00d4c0f7d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; @@ -77,4 +79,36 @@ public interface ErrorEventRepository extends EventRepository= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM error_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:method IS NULL OR e.e_method ILIKE concat('%', :method, '%')) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("method") String method, + @Param("error") String error); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java new file mode 100644 index 0000000000..c89fd6462f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.event; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.event.EventType; + +import javax.annotation.PostConstruct; +import java.util.concurrent.TimeUnit; + +@Component +public class EventPartitionConfiguration { + + @Value("${sql.events.partition_size:168}") + private int regularPartitionSizeInHours; + @Value("${sql.events.debug_partition_size:1}") + private int debugPartitionSizeInHours; + + private long regularPartitionSizeInMs; + private long debugPartitionSizeInMs; + + @PostConstruct + public void init() { + regularPartitionSizeInMs = TimeUnit.HOURS.toMillis(regularPartitionSizeInHours); + debugPartitionSizeInMs = TimeUnit.HOURS.toMillis(debugPartitionSizeInHours); + } + + public long getPartitionSizeInMs(EventType eventType) { + return eventType.isDebug() ? debugPartitionSizeInMs : regularPartitionSizeInMs; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index 46e0ff6662..19865cdc8b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -28,137 +28,12 @@ import org.thingsboard.server.dao.model.sql.EventEntity; import java.util.List; import java.util.UUID; -/** - * Created by Valerii Sosliuk on 5/3/2017. - */ public interface EventRepository, V extends Event> { List findLatestEvents(UUID tenantId, UUID entityId, int limit); Page findEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime, Pageable pageable); -// -// @Query(nativeQuery = true, -// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = :eventType " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + -// "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + -// "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + -// "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + -// "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + -// "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + -// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + -// "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + -// "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%')) ", -// countQuery = "SELECT count(*) FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = :eventType " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:type IS NULL OR lower(json_body->>'type') LIKE concat('%', lower(:type\\:\\:varchar), '%')) " + -// "AND (:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:entityName IS NULL OR lower(json_body->>'entityName') LIKE concat('%', lower(:entityName\\:\\:varchar), '%')) " + -// "AND (:relationType IS NULL OR lower(json_body->>'relationType') LIKE concat('%', lower(:relationType\\:\\:varchar), '%')) " + -// "AND (:bodyEntityId IS NULL OR lower(json_body->>'entityId') LIKE concat('%', lower(:bodyEntityId\\:\\:varchar), '%')) " + -// "AND (:msgType IS NULL OR lower(json_body->>'msgType') LIKE concat('%', lower(:msgType\\:\\:varchar), '%')) " + -// "AND ((:isError = FALSE) OR (json_body->>'error') IS NOT NULL) " + -// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%')) " + -// "AND (:data IS NULL OR lower(json_body->>'data') LIKE concat('%', lower(:data\\:\\:varchar), '%')) " + -// "AND (:metadata IS NULL OR lower(json_body->>'metadata') LIKE concat('%', lower(:metadata\\:\\:varchar), '%'))" -// ) -// Page findDebugRuleNodeEvents(@Param("tenantId") UUID tenantId, -// @Param("entityId") UUID entityId, -// @Param("entityType") String entityType, -// @Param("eventType") String eventType, -// @Param("startTime") Long startTime, -// @Param("endTime") Long endTime, -// @Param("type") String type, -// @Param("server") String server, -// @Param("entityName") String entityName, -// @Param("relationType") String relationType, -// @Param("bodyEntityId") String bodyEntityId, -// @Param("msgType") String msgType, -// @Param("isError") boolean isError, -// @Param("error") String error, -// @Param("data") String data, -// @Param("metadata") String metadata, -// Pageable pageable); -// -// @Query(nativeQuery = true, -// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = 'ERROR' " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + -// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))", -// countQuery = "SELECT count(*) FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = 'ERROR' " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:server IS NULL OR lower(json_body->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:method IS NULL OR lower(json_body->>'method') LIKE concat('%', lower(:method\\:\\:varchar), '%')) " + -// "AND (:error IS NULL OR lower(json_body->>'error') LIKE concat('%', lower(:error\\:\\:varchar), '%'))") -// Page findErrorEvents(@Param("tenantId") UUID tenantId, -// @Param("entityId") UUID entityId, -// @Param("entityType") String entityType, -// @Param("startTime") Long startTime, -// @Param("endTime") Long endTIme, -// @Param("server") String server, -// @Param("method") String method, -// @Param("error") String error, -// Pageable pageable); -// -// -// @Query(nativeQuery = true, -// value = "SELECT e.id, e.created_time, e.body, e.entity_id, e.entity_type, e.event_type, e.event_uid, e.tenant_id, ts FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = 'STATS' " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + -// "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ", -// countQuery = "SELECT count(*) FROM " + -// "(SELECT *, e.body\\:\\:jsonb as json_body FROM event e WHERE " + -// "e.tenant_id = :tenantId " + -// "AND e.entity_type = :entityType " + -// "AND e.entity_id = :entityId " + -// "AND e.event_type = 'LC_EVENT' " + -// "AND e.created_time >= :startTime AND (:endTime = 0 OR e.created_time <= :endTime) " + -// ") AS e WHERE " + -// "(:server IS NULL OR lower(e.body\\:\\:json->>'server') LIKE concat('%', lower(:server\\:\\:varchar), '%')) " + -// "AND (:messagesProcessed = 0 OR (json_body->>'messagesProcessed')\\:\\:integer >= :messagesProcessed) " + -// "AND (:errorsOccurred = 0 OR (json_body->>'errorsOccurred')\\:\\:integer >= :errorsOccurred) ") -// Page findStatisticsEvents(@Param("tenantId") UUID tenantId, -// @Param("entityId") UUID entityId, -// @Param("entityType") String entityType, -// @Param("startTime") Long startTime, -// @Param("endTime") Long endTIme, -// @Param("server") String server, -// @Param("messagesProcessed") Integer messagesProcessed, -// @Param("errorsOccurred") Integer errorsOccurred, -// Pageable pageable); -// + void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 0a39ab1be0..525150fd90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -48,13 +48,11 @@ import org.thingsboard.server.dao.timeseries.SqlPartition; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -65,12 +63,12 @@ import java.util.function.Function; @Component public class JpaBaseEventDao implements EventDao { - public static final long REGULAR_PARTITION_DURATION = TimeUnit.DAYS.toMillis(1); - public static final long DEBUG_PARTITION_DURATION = TimeUnit.HOURS.toMillis(1); - private final Map> partitionsByEventType = new ConcurrentHashMap<>(); private static final ReentrantLock partitionCreationLock = new ReentrantLock(); + @Autowired + private EventPartitionConfiguration partitionConfiguration; + @Autowired private SqlPartitioningRepository partitioningRepository; @@ -170,12 +168,12 @@ public class JpaBaseEventDao implements EventDao { } private void savePartitionIfNotExist(Event event) { - var partitionsMap = partitionsByEventType.get(event.getType()); - var partitionDuration = event.getType().isDebug() ? DEBUG_PARTITION_DURATION : REGULAR_PARTITION_DURATION; + EventType type = event.getType(); + var partitionsMap = partitionsByEventType.get(type); + var partitionDuration = partitionConfiguration.getPartitionSizeInMs(type); long partitionStartTs = event.getCreatedTime() - (event.getCreatedTime() % partitionDuration); if (partitionsMap.get(partitionStartTs) == null) { - savePartition(partitionsMap, new SqlPartition(event.getType().getTable(), partitionStartTs, - partitionStartTs + partitionDuration, Long.toString(partitionStartTs))); + savePartition(partitionsMap, new SqlPartition(type.getTable(), partitionStartTs, partitionStartTs + partitionDuration, Long.toString(partitionStartTs))); } } @@ -228,6 +226,42 @@ public class JpaBaseEventDao implements EventDao { } } + @Override + public void removeEvents(UUID tenantId, UUID entityId, Long startTime, Long endTime) { + log.debug("[{}][{}] Remove events [{}-{}] ", tenantId, entityId, startTime, endTime); + for (EventType eventType : EventType.values()) { + getEventRepository(eventType).removeEvents(tenantId, entityId, startTime, endTime); + } + } + + @Override + public void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime) { + if (eventFilter.isNotEmpty()) { + switch (eventFilter.getEventType()) { + case DEBUG_RULE_NODE: + removeEventsByFilter(tenantId, entityId, (RuleNodeDebugEventFilter) eventFilter, startTime, endTime); + break; + case DEBUG_RULE_CHAIN: + removeEventsByFilter(tenantId, entityId, (RuleChainDebugEventFilter) eventFilter, startTime, endTime); + break; + case LC_EVENT: + removeEventsByFilter(tenantId, entityId, (LifeCycleEventFilter) eventFilter, startTime, endTime); + break; + case ERROR: + removeEventsByFilter(tenantId, entityId, (ErrorEventFilter) eventFilter, startTime, endTime); + break; + case STATS: + removeEventsByFilter(tenantId, entityId, (StatisticsEventFilter) eventFilter, startTime, endTime); + break; + default: + throw new RuntimeException("Not supported event type: " + eventFilter.getEventType()); + } + } else { + getEventRepository(eventFilter.getEventType()).removeEvents(tenantId, entityId, startTime, endTime); + } + } + + private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, TimePageLink pageLink) { return DaoUtil.toPageData( ruleChainDebugEventRepository.findEvents( @@ -305,12 +339,88 @@ public class JpaBaseEventDao implements EventDao { pageLink.getStartTime(), pageLink.getEndTime(), eventFilter.getServer(), - eventFilter.getMessagesProcessed(), - eventFilter.getErrorsOccurred(), + eventFilter.getMinMessagesProcessed(), + eventFilter.getMaxMessagesProcessed(), + eventFilter.getMinErrorsOccurred(), + eventFilter.getMaxErrorsOccurred(), DaoUtil.toPageable(pageLink)) ); } + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, Long startTime, Long endTime) { + ruleChainDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMessage(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, RuleNodeDebugEventFilter eventFilter, Long startTime, Long endTime) { + parseUUID(eventFilter.getEntityId(), "Entity Id"); + parseUUID(eventFilter.getMsgId(), "Message Id"); + ruleNodeDebugEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMsgDirectionType(), + eventFilter.getEntityId(), + eventFilter.getEntityType(), + eventFilter.getMsgId(), + eventFilter.getMsgType(), + eventFilter.getRelationType(), + eventFilter.getDataSearch(), + eventFilter.getMetadataSearch(), + eventFilter.isError(), + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, ErrorEventFilter eventFilter, Long startTime, Long endTime) { + errorEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMethod(), + eventFilter.getErrorStr()); + + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, LifeCycleEventFilter eventFilter, Long startTime, Long endTime) { + boolean statusFilterEnabled = !StringUtils.isEmpty(eventFilter.getStatus()); + boolean statusFilter = statusFilterEnabled && eventFilter.getStatus().equalsIgnoreCase("Success"); + lcEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getEvent(), + statusFilterEnabled, + statusFilter, + eventFilter.getErrorStr()); + } + + private void removeEventsByFilter(UUID tenantId, UUID entityId, StatisticsEventFilter eventFilter, Long startTime, Long endTime) { + statsEventRepository.removeEvents( + tenantId, + entityId, + startTime, + endTime, + eventFilter.getServer(), + eventFilter.getMinMessagesProcessed(), + eventFilter.getMaxMessagesProcessed(), + eventFilter.getMinErrorsOccurred(), + eventFilter.getMaxErrorsOccurred() + ); + } + @Override public List findLatestEvents(UUID tenantId, UUID entityId, EventType eventType, int limit) { return DaoUtil.convertDataList(getEventRepository(eventType).findLatestEvents(tenantId, entityId, limit)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java index 298e57a1b9..e4a7f40256 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; @@ -77,4 +79,39 @@ public interface LifecycleEventRepository extends EventRepository= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM lc_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND ((:statusFilterEnabled = FALSE) OR e.e_success = :statusFilter) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String eventType, + @Param("statusFilterEnabled") boolean statusFilterEnabled, + @Param("statusFilter") boolean statusFilter, + @Param("error") String error); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java index 32d27765a5..d01a54fa30 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.RuleChainDebugEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; @@ -43,10 +45,10 @@ public interface RuleChainDebugEventRepository extends EventRepository findEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - Pageable pageable); + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + Pageable pageable); @Query(nativeQuery = true, value = "SELECT * FROM rule_chain_debug_event e WHERE " + @@ -70,13 +72,46 @@ public interface RuleChainDebugEventRepository extends EventRepository findEvents(@Param("tenantId") UUID tenantId, - @Param("entityId") UUID entityId, - @Param("startTime") Long startTime, - @Param("endTime") Long endTime, - @Param("serviceId") String server, - @Param("message") String message, - @Param("isError") boolean isError, - @Param("error") String error, - Pageable pageable); + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("message") String message, + @Param("isError") boolean isError, + @Param("error") String error, + Pageable pageable); + @Transactional + @Modifying + @Query("DELETE FROM RuleChainDebugEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM rule_chain_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:message IS NULL OR e.e_message ILIKE concat('%', :message, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("message") String message, + @Param("isError") boolean isError, + @Param("error") String error); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java index 6238acd5ac..98d404c36f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; @@ -100,4 +102,51 @@ public interface RuleNodeDebugEventRepository extends EventRepository= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM rule_node_debug_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:eventType IS NULL OR e.e_type ILIKE concat('%', :eventType, '%')) " + + "AND (:eventEntityId IS NULL OR e.e_entity_id = uuid(:eventEntityId)) " + + "AND (:eventEntityType IS NULL OR e.e_entity_type ILIKE concat('%', :eventEntityType, '%')) " + + "AND (:msgId IS NULL OR e.e_msg_id = uuid(:msgId)) " + + "AND (:msgType IS NULL OR e.e_msg_type ILIKE concat('%', :msgType, '%')) " + + "AND (:relationType IS NULL OR e.e_relation_type ILIKE concat('%', :relationType, '%')) " + + "AND (:data IS NULL OR e.e_data ILIKE concat('%', :data, '%')) " + + "AND (:metadata IS NULL OR e.e_metadata ILIKE concat('%', :metadata, '%')) " + + "AND ((:isError = FALSE) OR e.e_error IS NOT NULL) " + + "AND (:error IS NULL OR e.e_error ILIKE concat('%', :error, '%'))") + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("eventType") String type, + @Param("eventEntityId") String eventEntityId, + @Param("eventEntityType") String eventEntityType, + @Param("msgId") String eventMsgId, + @Param("msgType") String eventMsgType, + @Param("relationType") String relationType, + @Param("data") String data, + @Param("metadata") String metadata, + @Param("isError") boolean isError, + @Param("error") String error); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index 54b4565097..d8e5d6dcd5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -16,7 +16,7 @@ package org.thingsboard.server.dao.sql.event; import lombok.extern.slf4j.Slf4j; -import org.postgresql.util.PSQLException; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; @@ -28,8 +28,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import static org.thingsboard.server.dao.sql.event.JpaBaseEventDao.DEBUG_PARTITION_DURATION; -import static org.thingsboard.server.dao.sql.event.JpaBaseEventDao.REGULAR_PARTITION_DURATION; @Slf4j @Repository @@ -38,6 +36,9 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe private static final String SELECT_PARTITIONS_STMT = "SELECT tablename from pg_tables WHERE schemaname = 'public' and tablename like concat(?, '_%')"; private static final int PSQL_VERSION_14 = 140000; + @Autowired + private EventPartitionConfiguration partitionConfiguration; + private volatile Integer currentServerVersion; @Override @@ -50,7 +51,7 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe } private void cleanupEvents(EventType eventType, long eventExpTime) { - var partitionDuration = eventType.isDebug() ? DEBUG_PARTITION_DURATION : REGULAR_PARTITION_DURATION; + var partitionDuration = partitionConfiguration.getPartitionSizeInMs(eventType); List partitions = fetchPartitions(eventType); for (var partitionTs : partitions) { var partitionEndTs = partitionTs + partitionDuration; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java index 5eee96efbc..67116100da 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.StatisticsEvent; import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; @@ -51,8 +53,10 @@ public interface StatisticsEventRepository extends EventRepository= :startTime) " + "AND (:endTime IS NULL OR e.ts <= :endTime) " + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + - "AND (:messagesProcessed IS NULL OR e.e_messages_processed >= :messagesProcessed) " + - "AND (:errorsOccurred IS NULL OR e.e_errors_occurred >= :errorsOccurred)" + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" , countQuery = "SELECT count(*) FROM stats_event e WHERE " + "e.tenant_id = :tenantId " + @@ -60,16 +64,57 @@ public interface StatisticsEventRepository extends EventRepository= :startTime) " + "AND (:endTime IS NULL OR e.ts <= :endTime) " + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + - "AND (:messagesProcessed IS NULL OR e.e_messages_processed >= :messagesProcessed) " + - "AND (:errorsOccurred IS NULL OR e.e_errors_occurred >= :errorsOccurred)" + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" ) Page findEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("startTime") Long startTime, @Param("endTime") Long endTime, @Param("serviceId") String server, - @Param("messagesProcessed") Integer messagesProcessed, - @Param("errorsOccurred") Integer errorsOccurred, + @Param("minMessagesProcessed") Integer minMessagesProcessed, + @Param("maxMessagesProcessed") Integer maxMessagesProcessed, + @Param("minErrorsOccurred") Integer minErrorsOccurred, + @Param("maxErrorsOccurred") Integer maxErrorsOccurred, Pageable pageable); + @Transactional + @Modifying + @Query("DELETE FROM StatisticsEventEntity e WHERE " + + "e.tenantId = :tenantId " + + "AND e.entityId = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime)" + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime); + + @Transactional + @Modifying + @Query(nativeQuery = true, + value = "DELETE FROM stats_event e WHERE " + + "e.tenant_id = :tenantId " + + "AND e.entity_id = :entityId " + + "AND (:startTime IS NULL OR e.ts >= :startTime) " + + "AND (:endTime IS NULL OR e.ts <= :endTime) " + + "AND (:serviceId IS NULL OR e.service_id ILIKE concat('%', :serviceId, '%')) " + + "AND (:minMessagesProcessed IS NULL OR e.e_messages_processed >= :minMessagesProcessed) " + + "AND (:maxMessagesProcessed IS NULL OR e.e_messages_processed < :maxMessagesProcessed) " + + "AND (:minErrorsOccurred IS NULL OR e.e_errors_occurred >= :minErrorsOccurred) " + + "AND (:maxErrorsOccurred IS NULL OR e.e_errors_occurred < :maxErrorsOccurred)" + + ) + void removeEvents(@Param("tenantId") UUID tenantId, + @Param("entityId") UUID entityId, + @Param("startTime") Long startTime, + @Param("endTime") Long endTime, + @Param("serviceId") String server, + @Param("minMessagesProcessed") Integer minMessagesProcessed, + @Param("maxMessagesProcessed") Integer maxMessagesProcessed, + @Param("minErrorsOccurred") Integer minErrorsOccurred, + @Param("maxErrorsOccurred") Integer maxErrorsOccurred); } From 3e4cfdc37fa40b469e059459f0b35bb67efc7352 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 27 Jul 2022 15:45:12 +0300 Subject: [PATCH 08/13] UI: Add new filter params in Statistics events --- .../home/components/event/event-filter-panel.component.ts | 2 +- .../app/modules/home/components/event/event-table-config.ts | 6 ++++-- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts index 14227d6c17..68b803e006 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts @@ -75,7 +75,7 @@ export class EventFilterPanelComponent { } isNumberFields(key: string): string { - return ['messagesProcessed', 'errorsOccurred'].includes(key) ? key : ''; + return ['minMessagesProcessed', 'maxMessagesProcessed', 'minErrorsOccurred', 'maxErrorsOccurred'].includes(key) ? key : ''; } selectorValues(key: string): string[] { diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index c094e3696c..997e4db414 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -356,8 +356,10 @@ export class EventTableConfig extends EntityTableConfig { break; case EventType.STATS: this.filterColumns.push( - {key: 'messagesProcessed', title: 'event.min-messages-processed'}, - {key: 'errorsOccurred', title: 'event.min-errors-occurred'} + {key: 'minMessagesProcessed', title: 'event.min-messages-processed'}, + {key: 'maxMessagesProcessed', title: 'event.max-messages-processed'}, + {key: 'minErrorsOccurred', title: 'event.min-errors-occurred'}, + {key: 'maxErrorsOccurred', title: 'event.max-errors-occurred'} ); break; case DebugEventType.DEBUG_RULE_NODE: diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0ea7d08742..6a7226860d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2024,8 +2024,10 @@ "success": "Success", "failed": "Failed", "messages-processed": "Messages processed", + "max-messages-processed": "Maximum messages processed", "min-messages-processed": "Minimum messages processed", "errors-occurred": "Errors occurred", + "max-errors-occurred": "Maximum errors occurred", "min-errors-occurred": "Minimum errors occurred", "min-value": "Minimum value is 0.", "all-events": "All", From a38edcd62b54ce96bb121f5e9cdb10d1d3d501a6 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 27 Jul 2022 18:56:09 +0300 Subject: [PATCH 09/13] Upgrade script for the events migration --- .../main/data/upgrade/3.4.0/schema_update.sql | 215 ++++++++++++++++++ .../install/ThingsboardInstallService.java | 3 + .../install/SqlDatabaseUpgradeService.java | 12 + .../service/ttl/EventsCleanUpService.java | 8 +- .../server/dao/event/EventService.java | 3 +- .../server/common/data/event/ErrorEvent.java | 15 +- .../common/data/event/RuleNodeDebugEvent.java | 3 +- .../common/data/event/StatisticsEvent.java | 6 +- .../server/dao/event/BaseEventService.java | 4 +- .../server/dao/event/EventDao.java | 7 +- .../dao/sql/event/ErrorEventRepository.java | 4 +- .../server/dao/sql/event/JpaBaseEventDao.java | 21 +- .../sql/event/LifecycleEventRepository.java | 5 +- .../event/RuleChainDebugEventRepository.java | 4 +- .../event/RuleNodeDebugEventRepository.java | 5 +- .../sql/event/StatisticsEventRepository.java | 4 +- .../dao/service/AbstractServiceTest.java | 2 +- .../service/event/BaseEventServiceTest.java | 77 +++---- .../dao/sql/event/JpaBaseEventDaoTest.java | 214 +++++++---------- 19 files changed, 409 insertions(+), 203 deletions(-) create mode 100644 application/src/main/data/upgrade/3.4.0/schema_update.sql diff --git a/application/src/main/data/upgrade/3.4.0/schema_update.sql b/application/src/main/data/upgrade/3.4.0/schema_update.sql new file mode 100644 index 0000000000..2b4b5af42d --- /dev/null +++ b/application/src/main/data/upgrade/3.4.0/schema_update.sql @@ -0,0 +1,215 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS rule_node_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL , + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar, + e_type varchar, + e_entity_id uuid, + e_entity_type varchar, + e_msg_id uuid, + e_msg_type varchar, + e_data_type varchar, + e_relation_type varchar, + e_data varchar, + e_metadata varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS rule_chain_debug_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_message varchar, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS stats_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_messages_processed bigint NOT NULL, + e_errors_occurred bigint NOT NULL +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS lc_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_type varchar NOT NULL, + e_success boolean NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS error_event ( + id uuid NOT NULL, + tenant_id uuid NOT NULL, + ts bigint NOT NULL, + entity_id uuid NOT NULL, + service_id varchar NOT NULL, + e_method varchar NOT NULL, + e_error varchar +) PARTITION BY RANGE (ts); + +CREATE INDEX IF NOT EXISTS idx_rule_node_debug_event_main + ON rule_node_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_debug_event_main + ON rule_chain_debug_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_stats_event_main + ON stats_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_lc_event_main + ON lc_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + +CREATE INDEX IF NOT EXISTS idx_error_event_main + ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); + + +-- Useful to migrate old events to the new table structure; +CREATE OR REPLACE PROCEDURE migrate_regular_events(IN start_ts_in_ms bigint, IN partition_size_in_hours int) + LANGUAGE plpgsql AS +$$ +DECLARE + partition_size_in_ms bigint; + p record; + table_name varchar; +BEGIN + partition_size_in_ms = partition_size_in_hours * 3600 * 1000; + + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('STATS', 'LC_EVENT', 'ERROR') and ts > start_ts_in_ms + LOOP + IF p.event_type = 'STATS' THEN + table_name := 'stats_event'; + ELSEIF p.event_type = 'LC_EVENT' THEN + table_name := 'lc_event'; + ELSEIF p.event_type = 'ERROR' THEN + table_name := 'error_event'; + END IF; + RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms); + EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms)); + END LOOP; + + INSERT INTO stats_event + SELECT id, + tenant_id, + ts, + entity_id, + body::json ->> 'server', + (body::json ->> 'messagesProcessed')::bigint, + (body::json ->> 'errorsOccurred')::bigint + FROM event + WHERE ts > start_ts_in_ms + AND event_type = 'STATS' + ON CONFLICT DO NOTHING; + + INSERT INTO lc_event + SELECT id, + tenant_id, + ts, + entity_id, + body::json ->> 'server', + body::json ->> 'event', + (body::json ->> 'success')::boolean, + body::json ->> 'error' + FROM event + WHERE ts > start_ts_in_ms + AND event_type = 'LC_EVENT' + ON CONFLICT DO NOTHING; + + INSERT INTO error_event + SELECT id, + tenant_id, + ts, + entity_id, + body::json ->> 'server', + body::json ->> 'method', + body::json ->> 'error' + FROM event + WHERE ts > start_ts_in_ms + AND event_type = 'ERROR' + ON CONFLICT DO NOTHING; + +END +$$; + +-- Useful to migrate old debug events to the new table structure; +CREATE OR REPLACE PROCEDURE migrate_debug_events(IN start_ts_in_ms bigint, IN partition_size_in_hours int) + LANGUAGE plpgsql AS +$$ +DECLARE + partition_size_in_ms bigint; + p record; + table_name varchar; +BEGIN + partition_size_in_ms = partition_size_in_hours * 3600 * 1000; + + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') and ts > start_ts_in_ms + LOOP + IF p.event_type = 'DEBUG_RULE_NODE' THEN + table_name := 'rule_node_debug_event'; + ELSEIF p.event_type = 'DEBUG_RULE_CHAIN' THEN + table_name := 'rule_chain_debug_event'; + END IF; + RAISE NOTICE '[%] Partition to create : [%-%]', table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms); + EXECUTE format('CREATE TABLE IF NOT EXISTS %s_%s PARTITION OF %s FOR VALUES FROM ( %s ) TO ( %s )', table_name, p.partition_ts, table_name, p.partition_ts, (p.partition_ts + partition_size_in_ms)); + END LOOP; + + INSERT INTO rule_node_debug_event + SELECT id, + tenant_id, + ts, + entity_id, + body::json ->> 'server', + body::json ->> 'type', + (body::json ->> 'entityId')::uuid, + body::json ->> 'entityName', + (body::json ->> 'msgId')::uuid, + body::json ->> 'msgType', + body::json ->> 'dataType', + body::json ->> 'relationType', + body::json ->> 'data', + body::json ->> 'metadata', + body::json ->> 'error' + FROM event + WHERE ts > start_ts_in_ms + AND event_type = 'DEBUG_RULE_NODE' + ON CONFLICT DO NOTHING; + + INSERT INTO rule_chain_debug_event + SELECT id, + tenant_id, + ts, + entity_id, + body::json ->> 'server', + body::json ->> 'message', + body::json ->> 'error' + FROM event + WHERE ts > start_ts_in_ms + AND event_type = 'DEBUG_RULE_CHAIN' + ON CONFLICT DO NOTHING; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 40e15c7b0e..ea7f76e1c3 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -221,6 +221,9 @@ public class ThingsboardInstallService { log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); dataUpdateService.updateData("3.3.4"); + case "3.4.0": + log.info("Upgrading ThingsBoard from version 3.4.0 to 3.4.1 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.4.0"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index d22039683d..b8cdf491d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -596,6 +596,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.4.0": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.4.0", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004001;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index 08ee57119c..3dc66b6210 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -25,7 +25,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import java.util.concurrent.TimeUnit; -@TbCoreComponent @Slf4j @Service public class EventsCleanUpService extends AbstractCleanUpService { @@ -39,9 +38,6 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Value("${sql.ttl.events.debug_events_ttl}") private long debugTtlInSec; - @Value("${sql.ttl.events.execution_interval_ms}") - private long executionIntervalInMs; - @Value("${sql.ttl.events.enabled}") private boolean ttlTaskExecutionEnabled; @@ -54,11 +50,11 @@ public class EventsCleanUpService extends AbstractCleanUpService { @Scheduled(initialDelayString = RANDOM_DELAY_INTERVAL_MS_EXPRESSION, fixedDelayString = "${sql.ttl.events.execution_interval_ms}") public void cleanUp() { - if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { + if (ttlTaskExecutionEnabled) { long ts = System.currentTimeMillis(); long regularEventExpTs = ttlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(ttlInSec) : 0; long debugEventExpTs = debugTtlInSec > 0 ? ts - TimeUnit.SECONDS.toMillis(debugTtlInSec) : 0; - eventService.cleanupEvents(regularEventExpTs, debugEventExpTs); + eventService.cleanupEvents(regularEventExpTs, debugEventExpTs, isSystemTenantPartitionMine()); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 2b89959b56..feed84598b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; import java.util.List; -import java.util.Optional; public interface EventService { @@ -42,6 +41,6 @@ public interface EventService { void removeEvents(TenantId tenantId, EntityId entityId, EventFilter eventFilter, Long startTime, Long endTime); - void cleanupEvents(long regularEventExpTs, long debugEventExpTs); + void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java index f811a118cf..341548df59 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -40,9 +41,11 @@ public class ErrorEvent extends Event { this.error = error; } - @Getter @Setter + @Getter + @Setter private String method; - @Getter @Setter + @Getter + @Setter private String error; @Override @@ -52,6 +55,12 @@ public class ErrorEvent extends Event { @Override public EventInfo toInfo(EntityType entityType) { - return null; + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("method", method); + if (error != null) { + json.put("error", error); + } + return eventInfo; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java index fdca84239d..ee87d976cf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -84,7 +84,8 @@ public class RuleNodeDebugEvent extends Event { var json = (ObjectNode) eventInfo.getBody(); json.put("type", eventType); if (eventEntity != null) { - json.put("entityId", eventEntity.getId().toString()).put("entityType", eventEntity.getEntityType().name()); + json.put("entityId", eventEntity.getId().toString()) + .put("entityType", eventEntity.getEntityType().name()); } if (msgId != null) { json.put("msgId", msgId.toString()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java index b574949cf8..0e5de8263f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -51,6 +52,9 @@ public class StatisticsEvent extends Event { @Override public EventInfo toInfo(EntityType entityType) { - return null; + EventInfo eventInfo = super.toInfo(entityType); + var json = (ObjectNode) eventInfo.getBody(); + json.put("messagesProcessed", messagesProcessed).put("errorsOccurred", errorsOccurred); + return eventInfo; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index fad7f6b310..f07a31a05e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -125,8 +125,8 @@ public class BaseEventService implements EventService { } @Override - public void cleanupEvents(long regularEventExpTs, long debugEventExpTs) { - eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs); + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) { + eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs, cleanupDb); } private PageData convert(EntityType entityType, PageData pd) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index e09ea59872..e41668f699 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -16,15 +16,11 @@ package org.thingsboard.server.dao.event; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.event.EventType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.dao.Dao; import java.util.List; import java.util.UUID; @@ -70,8 +66,9 @@ public interface EventDao { * Executes stored procedure to cleanup old events. Uses separate ttl for debug and other events. * @param regularEventExpTs the expiration time of the regular events * @param debugEventExpTs the expiration time of the debug events + * @param cleanupDb */ - void cleanupEvents(long regularEventExpTs, long debugEventExpTs); + void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb); /** * Removes all events for the specified entity and time interval diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java index c00d4c0f7d..43b4d0c91d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/ErrorEventRepository.java @@ -35,8 +35,8 @@ import java.util.UUID; public interface ErrorEventRepository extends EventRepository, JpaRepository { @Override - @Query("SELECT e FROM ErrorEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") - List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query(nativeQuery = true, value = "SELECT * FROM error_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override @Query("SELECT e FROM ErrorEventEntity e WHERE " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 525150fd90..907aad0a50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -427,14 +427,29 @@ public class JpaBaseEventDao implements EventDao { } @Override - public void cleanupEvents(long regularEventExpTs, long debugEventExpTs) { + public void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb) { if (regularEventExpTs > 0) { log.info("Going to cleanup regular events with exp time: {}", regularEventExpTs); - eventCleanupRepository.cleanupEvents(regularEventExpTs, false); + if (cleanupDb) { + eventCleanupRepository.cleanupEvents(regularEventExpTs, false); + } + cleanupPartitions(regularEventExpTs, false); } if (debugEventExpTs > 0) { log.info("Going to cleanup debug events with exp time: {}", debugEventExpTs); - eventCleanupRepository.cleanupEvents(debugEventExpTs, true); + if (cleanupDb) { + eventCleanupRepository.cleanupEvents(debugEventExpTs, true); + } + cleanupPartitions(debugEventExpTs, true); + } + } + + private void cleanupPartitions(long expTime, boolean isDebug) { + for (EventType eventType : EventType.values()) { + if (eventType.isDebug() == isDebug) { + Map partitions = partitionsByEventType.get(eventType); + partitions.keySet().removeIf(startTs -> startTs + partitionConfiguration.getPartitionSizeInMs(eventType) < expTime); + } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java index e4a7f40256..11215e4771 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -24,6 +24,7 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.event.LifecycleEvent; import org.thingsboard.server.dao.model.sql.LifecycleEventEntity; +import org.thingsboard.server.dao.model.sql.RuleChainDebugEventEntity; import java.util.List; import java.util.UUID; @@ -31,8 +32,8 @@ import java.util.UUID; public interface LifecycleEventRepository extends EventRepository, JpaRepository { @Override - @Query("SELECT e FROM LifecycleEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") - List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query(nativeQuery = true, value = "SELECT * FROM lc_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Query("SELECT e FROM LifecycleEventEntity e WHERE " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java index d01a54fa30..eb8ba01cf0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleChainDebugEventRepository.java @@ -34,8 +34,8 @@ import java.util.UUID; public interface RuleChainDebugEventRepository extends EventRepository, JpaRepository { @Override - @Query("SELECT e FROM RuleChainDebugEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") - List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query(nativeQuery = true, value = "SELECT * FROM rule_chain_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override @Query("SELECT e FROM RuleChainDebugEventEntity e WHERE " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java index 98d404c36f..3df9989674 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/RuleNodeDebugEventRepository.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.event.ErrorEvent; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; import org.thingsboard.server.dao.model.sql.ErrorEventEntity; import org.thingsboard.server.dao.model.sql.RuleNodeDebugEventEntity; +import org.thingsboard.server.dao.model.sql.StatisticsEventEntity; import java.util.List; import java.util.UUID; @@ -34,8 +35,8 @@ import java.util.UUID; public interface RuleNodeDebugEventRepository extends EventRepository, JpaRepository { @Override - @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") - List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Override @Query("SELECT e FROM RuleNodeDebugEventEntity e WHERE " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java index 67116100da..3d2c4f4516 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/StatisticsEventRepository.java @@ -31,8 +31,8 @@ import java.util.UUID; public interface StatisticsEventRepository extends EventRepository, JpaRepository { @Override - @Query("SELECT e FROM LifecycleEventEntity e WHERE e.tenantId = :tenantId AND e.entityId = :entityId ORDER BY e.ts DESC") - List findLatestEvents(UUID tenantId, UUID entityId, int limit); + @Query(nativeQuery = true, value = "SELECT * FROM stats_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Query("SELECT e FROM StatisticsEventEntity e WHERE " + "e.tenantId = :tenantId " + diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 3c856c2c70..475f84a29a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -188,7 +188,7 @@ public abstract class AbstractServiceTest { } - protected RuleNodeDebugEvent generateEvent(TenantId tenantId, EntityId entityId, String eventType, String eventUid) throws IOException { + protected RuleNodeDebugEvent generateEvent(TenantId tenantId, EntityId entityId) throws IOException { if (tenantId == null) { tenantId = TenantId.fromUUID(Uuids.timeBased()); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java index 4bb315e5dc..26e0df8c6d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/event/BaseEventServiceTest.java @@ -19,7 +19,6 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.event.Event; import org.thingsboard.server.common.data.event.EventType; @@ -35,7 +34,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.service.AbstractServiceTest; import java.text.ParseException; -import java.util.Optional; +import java.util.List; import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; @@ -57,16 +56,14 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { @Test public void saveEvent() throws Exception { + TenantId tenantId = new TenantId(Uuids.timeBased()); DeviceId devId = new DeviceId(Uuids.timeBased()); - RuleNodeDebugEvent event = generateEvent(null, devId, "ALARM", Uuids.timeBased().toString()); + RuleNodeDebugEvent event = generateEvent(tenantId, devId); eventService.saveAsync(event).get(); - throw new RuntimeException("fix me!"); -// Optional loaded = eventService.findEvent(event.getTenantId(), event.getEntityId(), event.getType(), event.getUid()); -// Assert.assertTrue(loaded.isPresent()); -// Assert.assertNotNull(loaded.get()); -// Assert.assertEquals(event.getEntityId(), loaded.get().getEntityId()); -// Assert.assertEquals(event.getType(), loaded.get().getType()); -// Assert.assertEquals(event.getBody(), loaded.get().getBody()); + List loaded = eventService.findLatestEvents(event.getTenantId(), devId, event.getType(), 1); + Assert.assertNotNull(loaded); + Assert.assertEquals(1, loaded.size()); + Assert.assertEquals(event.getData(), loaded.get(0).getBody().get("data").asText()); } @Test @@ -74,29 +71,29 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); - EventInfo savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - EventInfo savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); - EventInfo savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); + Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); + Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); - TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime"), startTime, endTime); + TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts"), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 2); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); - Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); + Assert.assertEquals(2, events.getData().size()); + Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId()); + Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId()); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 1); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); + Assert.assertEquals(1, events.getData().size()); + Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId()); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true); } @Test @@ -104,36 +101,36 @@ public abstract class BaseEventServiceTest extends AbstractServiceTest { CustomerId customerId = new CustomerId(Uuids.timeBased()); TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); saveEventWithProvidedTime(timeBeforeStartTime, customerId, tenantId); - EventInfo savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); - EventInfo savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); - EventInfo savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); + Event savedEvent = saveEventWithProvidedTime(eventTime, customerId, tenantId); + Event savedEvent2 = saveEventWithProvidedTime(eventTime + 1, customerId, tenantId); + Event savedEvent3 = saveEventWithProvidedTime(eventTime + 2, customerId, tenantId); saveEventWithProvidedTime(timeAfterEndTime, customerId, tenantId); - TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("createdTime", SortOrder.Direction.DESC), startTime, endTime); + TimePageLink timePageLink = new TimePageLink(2, 0, "", new SortOrder("ts", SortOrder.Direction.DESC), startTime, endTime); - PageData events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink); + PageData events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 2); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent3.getUuidId())); - Assert.assertTrue(events.getData().get(1).getUuidId().equals(savedEvent2.getUuidId())); + Assert.assertEquals(2, events.getData().size()); + Assert.assertEquals(savedEvent3.getUuidId(), events.getData().get(0).getUuidId()); + Assert.assertEquals(savedEvent2.getUuidId(), events.getData().get(1).getUuidId()); Assert.assertTrue(events.hasNext()); - events = eventService.findEvents(tenantId, customerId, EventType.STATS, timePageLink.nextPageLink()); + events = eventService.findEvents(tenantId, customerId, EventType.DEBUG_RULE_NODE, timePageLink.nextPageLink()); Assert.assertNotNull(events.getData()); - Assert.assertTrue(events.getData().size() == 1); - Assert.assertTrue(events.getData().get(0).getUuidId().equals(savedEvent.getUuidId())); + Assert.assertEquals(1, events.getData().size()); + Assert.assertEquals(savedEvent.getUuidId(), events.getData().get(0).getUuidId()); Assert.assertFalse(events.hasNext()); - eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1); + eventService.cleanupEvents(timeBeforeStartTime - 1, timeAfterEndTime + 1, true); } - private EventInfo saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { - throw new RuntimeException("fix me!"); -// EventInfo event = generateEvent(tenantId, entityId, DataConstants.STATS, null); -// event.setId(new EventId(Uuids.startOf(time))); -// eventService.saveAsync(event).get(); -// return event; + private Event saveEventWithProvidedTime(long time, EntityId entityId, TenantId tenantId) throws Exception { + RuleNodeDebugEvent event = generateEvent(tenantId, entityId); + event.setId(new EventId(Uuids.timeBased())); + event.setCreatedTime(time); + eventService.saveAsync(event).get(); + return event; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java index 16170c10ea..b2091379b4 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDaoTest.java @@ -17,16 +17,26 @@ package org.thingsboard.server.dao.sql.event; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.extern.slf4j.Slf4j; -import org.junit.After; +import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.StatisticsEvent; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.AbstractJpaDaoTest; import org.thingsboard.server.dao.event.EventDao; +import java.util.List; import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; -/** - * Created by Valerii Sosliuk on 5/5/2017. - */ @Slf4j public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { @@ -34,131 +44,77 @@ public class JpaBaseEventDaoTest extends AbstractJpaDaoTest { private EventDao eventDao; UUID tenantId = Uuids.timeBased(); - @After - public void deleteEvents() { - throw new RuntimeException("fix me!"); -// List events = eventDao.find(TenantId.fromUUID(tenantId)); -// for (EventInfo event : events) { -// eventDao.removeById(TenantId.fromUUID(tenantId), event.getUuidId()); -// } + + @Test + public void findEvent() throws InterruptedException, ExecutionException, TimeoutException { + UUID entityId = Uuids.timeBased(); + + Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId); + eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES); + Thread.sleep(2); + Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId); + eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES); + + List foundEvents = eventDao.findLatestEvents(tenantId, entityId, EventType.STATS, 1); + assertNotNull("Events expected to be not null", foundEvents); + assertEquals(1, foundEvents.size()); + assertEquals(event2, foundEvents.get(0)); + } + + @Test + public void findEventsByEntityIdAndPageLink() throws Exception { + UUID entityId1 = Uuids.timeBased(); + UUID entityId2 = Uuids.timeBased(); + long startTime = System.currentTimeMillis(); + + Event event1 = getStatsEvent(Uuids.timeBased(), tenantId, entityId1); + eventDao.saveAsync(event1).get(1, TimeUnit.MINUTES); + Thread.sleep(2); + Event event2 = getStatsEvent(Uuids.timeBased(), tenantId, entityId2); + eventDao.saveAsync(event2).get(1, TimeUnit.MINUTES); + + long endTime = System.currentTimeMillis(); + + PageData events1 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, new TimePageLink(30)); + assertEquals(1, events1.getData().size()); + + PageData events2 = eventDao.findEvents(tenantId, entityId2, EventType.STATS, new TimePageLink(30)); + assertEquals(1, events2.getData().size()); + + PageData events3 = eventDao.findEvents(tenantId, Uuids.timeBased(), EventType.STATS, new TimePageLink(30)); + assertEquals(0, events3.getData().size()); + + + TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); + PageData events12 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink2); + assertEquals(1, events12.getData().size()); + assertEquals(event1, events12.getData().get(0)); + + TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); + PageData events13 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink3); + assertEquals(1, events13.getData().size()); + assertEquals(event1, events13.getData().get(0)); + + TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); + PageData events14 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4); + assertEquals(1, events14.getData().size()); + assertEquals(event1, events14.getData().get(0)); + + pageLink4 = pageLink4.nextPageLink(); + PageData events6 = eventDao.findEvents(tenantId, entityId1, EventType.STATS, pageLink4); + assertEquals(0, events6.getData().size()); + } -// @Test -// public void findEvent() { -// UUID entityId = Uuids.timeBased(); -// EventInfo savedEvent = eventDao.save(TenantId.fromUUID(tenantId), getEvent(entityId, tenantId, entityId)); -// EventInfo foundEvent = eventDao.findEvent(tenantId, new DeviceId(entityId), DataConstants.STATS, savedEvent.getUid()); -// assertNotNull("Event expected to be not null", foundEvent); -// assertEquals(savedEvent.getId(), foundEvent.getId()); -// } -// -// @Test -// public void findEventsByEntityIdAndPageLink() throws Exception { -// UUID entityId1 = Uuids.timeBased(); -// UUID entityId2 = Uuids.timeBased(); -// long startTime = System.currentTimeMillis(); -// long endTime = createEventsTwoEntities(tenantId, entityId1, entityId2, 20); -// -// TimePageLink pageLink1 = new TimePageLink(30); -// PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink1); -// assertEquals(10, events1.getData().size()); -// -// TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); -// PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink2); -// assertEquals(10, events2.getData().size()); -// -// TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); -// PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink3); -// assertEquals(10, events3.getData().size()); -// -// TimePageLink pageLink4 = new TimePageLink(5, 0, "", null, startTime, endTime); -// PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); -// assertEquals(5, events4.getData().size()); -// -// pageLink4 = pageLink4.nextPageLink(); -// PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); -// assertEquals(5, events5.getData().size()); -// -// pageLink4 = pageLink4.nextPageLink(); -// PageData events6 = eventDao.findEvents(tenantId, new DeviceId(entityId1), pageLink4); -// assertEquals(0, events6.getData().size()); -// -// } -// -// @Test -// public void findEventsByEntityIdAndEventTypeAndPageLink() throws Exception { -// UUID entityId1 = Uuids.timeBased(); -// UUID entityId2 = Uuids.timeBased(); -// long startTime = System.currentTimeMillis(); -// long endTime = createEventsTwoEntitiesTwoTypes(tenantId, entityId1, entityId2, 20); -// -// TimePageLink pageLink1 = new TimePageLink(30); -// PageData events1 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink1); -// assertEquals(5, events1.getData().size()); -// -// TimePageLink pageLink2 = new TimePageLink(30, 0, "", null, startTime, null); -// PageData events2 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink2); -// assertEquals(5, events2.getData().size()); -// -// TimePageLink pageLink3 = new TimePageLink(30, 0, "", null, startTime, endTime); -// PageData events3 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink3); -// assertEquals(5, events3.getData().size()); -// -// TimePageLink pageLink4 = new TimePageLink(4, 0, "", null, startTime, endTime); -// PageData events4 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); -// assertEquals(4, events4.getData().size()); -// -// pageLink4 = pageLink4.nextPageLink(); -// PageData events5 = eventDao.findEvents(tenantId, new DeviceId(entityId1), ALARM, pageLink4); -// assertEquals(1, events5.getData().size()); -// } -// -// private long createEventsTwoEntitiesTwoTypes(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { -// for (int i = 0; i < count / 2; i++) { -// String type = i % 2 == 0 ? STATS : ALARM; -// UUID eventId1 = Uuids.timeBased(); -// EventInfo event1 = getEvent(eventId1, tenantId, entityId1, type); -// eventDao.saveAsync(event1).get(); -// UUID eventId2 = Uuids.timeBased(); -// EventInfo event2 = getEvent(eventId2, tenantId, entityId2, type); -// eventDao.saveAsync(event2).get(); -// } -// return System.currentTimeMillis(); -// } -// -// private long createEventsTwoEntities(UUID tenantId, UUID entityId1, UUID entityId2, int count) throws Exception { -// for (int i = 0; i < count / 2; i++) { -// UUID eventId1 = Uuids.timeBased(); -// EventInfo event1 = getEvent(eventId1, tenantId, entityId1); -// eventDao.saveAsync(event1).get(); -// UUID eventId2 = Uuids.timeBased(); -// EventInfo event2 = getEvent(eventId2, tenantId, entityId2); -// eventDao.saveAsync(event2).get(); -// } -// return System.currentTimeMillis(); -// } -// -// private EventInfo getEvent(UUID eventId, UUID tenantId, UUID entityId, String type) { -// EventInfo event = getEvent(eventId, tenantId, entityId); -// event.setType(type); -// return event; -// } -// -// private EventInfo getEvent(UUID eventId, UUID tenantId, UUID entityId) { -// EventInfo event = new EventInfo(); -// event.setId(new EventId(eventId)); -// event.setTenantId(TenantId.fromUUID(tenantId)); -// EntityId deviceId = new DeviceId(entityId); -// event.setEntityId(deviceId); -// event.setUid(event.getId().getId().toString()); -// event.setType(STATS); -// ObjectMapper mapper = new ObjectMapper(); -// try { -// JsonNode jsonNode = mapper.readTree("{\"key\":\"value\"}"); -// event.setBody(jsonNode); -// } catch (IOException e) { -// log.error(e.getMessage(), e); -// } -// return event; -// } + private Event getStatsEvent(UUID eventId, UUID tenantId, UUID entityId) { + StatisticsEvent.StatisticsEventBuilder event = StatisticsEvent.builder(); + event.id(eventId); + event.ts(System.currentTimeMillis()); + event.tenantId(new TenantId(tenantId)); + event.entityId(entityId); + event.serviceId("server A"); + event.messagesProcessed(1); + event.errorsOccurred(0); + return event.build(); + } } From 07de58fe2185f344860c0600cacc67c65fb6bec9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 27 Jul 2022 19:22:10 +0300 Subject: [PATCH 10/13] Automatic migration of the events during upgrade --- .../install/ThingsboardInstallService.java | 1 + .../install/SqlDatabaseUpgradeService.java | 2 +- .../update/DefaultDataUpdateService.java | 11 ++++++++++- .../server/dao/event/EventService.java | 1 + .../server/dao/event/BaseEventService.java | 10 ++++++++++ .../thingsboard/server/dao/event/EventDao.java | 2 ++ .../dao/sql/event/EventCleanupRepository.java | 1 + .../sql/event/EventPartitionConfiguration.java | 3 +++ .../server/dao/sql/event/JpaBaseEventDao.java | 4 ++++ .../sql/event/SqlEventCleanupRepository.java | 17 +++++++++++++++++ 10 files changed, 50 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ea7f76e1c3..2c3251834f 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -224,6 +224,7 @@ public class ThingsboardInstallService { case "3.4.0": log.info("Upgrading ThingsBoard from version 3.4.0 to 3.4.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.4.0"); + dataUpdateService.updateData("3.4.0"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b8cdf491d7..ea976b2dec 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -25,6 +25,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.QueueId; @@ -678,5 +679,4 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService } - } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b90da586fc..d0d0322e0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -58,6 +58,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.relation.RelationService; @@ -128,6 +129,9 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private SystemDataLoaderService systemDataLoaderService; + @Autowired + private EventService eventService; + @Override public void updateData(String fromVersion) throws Exception { switch (fromVersion) { @@ -159,6 +163,11 @@ public class DefaultDataUpdateService implements DataUpdateService { tenantsProfileQueueConfigurationUpdater.updateEntities(); rateLimitsUpdater.updateEntities(); break; + case "3.4.0": + if (System.getProperty("TB_EVENTS_MIGRATION", "false").equalsIgnoreCase("true")) { + log.info("Updating data from version 3.3.4 to 3.4.0 ..."); + eventService.migrateEvents(); + } default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } @@ -602,7 +611,7 @@ public class DefaultDataUpdateService implements DataUpdateService { }); } } catch (Exception e) { - log.error("Failed to update tenant profile queue configuration name=["+profile.getName()+"], id=["+ profile.getId().getId() +"]", e); + log.error("Failed to update tenant profile queue configuration name=[" + profile.getName() + "], id=[" + profile.getId().getId() + "]", e); } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index feed84598b..3b0de80811 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -43,4 +43,5 @@ public interface EventService { void cleanupEvents(long regularEventExpTs, long debugEventExpTs, boolean cleanupDb); + void migrateEvents(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index f07a31a05e..88d68fa574 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -45,6 +45,11 @@ import java.util.stream.Collectors; @Slf4j public class BaseEventService implements EventService { + @Value("${sql.ttl.events.events_ttl:0}") + private long ttlInSec; + @Value("${sql.ttl.events.debug_events_ttl:604800}") + private long debugTtlInSec; + @Value("${event.debug.max-symbols:4096}") private int maxDebugEventSymbols; @@ -129,6 +134,11 @@ public class BaseEventService implements EventService { eventDao.cleanupEvents(regularEventExpTs, debugEventExpTs, cleanupDb); } + @Override + public void migrateEvents() { + eventDao.migrateEvents(ttlInSec > 0 ? System.currentTimeMillis() - ttlInSec : 0, debugTtlInSec > 0 ? System.currentTimeMillis() - debugTtlInSec : 0); + } + private PageData convert(EntityType entityType, PageData pd) { return new PageData<>(pd.getData() == null ? null : pd.getData().stream().map(e -> e.toInfo(entityType)).collect(Collectors.toList()) diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index e41668f699..7d26fabe62 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -91,4 +91,6 @@ public interface EventDao { * @param endTime */ void removeEvents(UUID tenantId, UUID entityId, EventFilter eventFilter, Long startTime, Long endTime); + + void migrateEvents(long regularEventTs, long debugEventTs); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java index 8631db46ff..dd982727a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventCleanupRepository.java @@ -19,4 +19,5 @@ public interface EventCleanupRepository { void cleanupEvents(long eventExpTime, boolean debug); + void migrateEvents(long regularEventTs, long debugEventTs); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java index c89fd6462f..ef49ee9521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventPartitionConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.sql.event; +import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.event.EventType; @@ -25,8 +26,10 @@ import java.util.concurrent.TimeUnit; @Component public class EventPartitionConfiguration { + @Getter @Value("${sql.events.partition_size:168}") private int regularPartitionSizeInHours; + @Getter @Value("${sql.events.debug_partition_size:1}") private int debugPartitionSizeInHours; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index 907aad0a50..bfea45307f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -261,6 +261,10 @@ public class JpaBaseEventDao implements EventDao { } } + @Override + public void migrateEvents(long regularEventTs, long debugEventTs) { + eventCleanupRepository.migrateEvents(regularEventTs, debugEventTs); + } private PageData findEventByFilter(UUID tenantId, UUID entityId, RuleChainDebugEventFilter eventFilter, TimePageLink pageLink) { return DaoUtil.toPageData( diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index d8e5d6dcd5..cdc8b5e348 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -50,6 +50,23 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe } } + @Override + public void migrateEvents(long regularEventTs, long debugEventTs) { + callMigrateFunction("migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours()); + callMigrateFunction("migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours()); + } + + private void callMigrateFunction(String functionName, long startTs, int partitionSizeInHours) { + try (Connection connection = dataSource.getConnection(); + PreparedStatement stmt = connection.prepareStatement("call " + functionName + "(?,?)")) { + stmt.setLong(1, startTs); + stmt.setInt(2, partitionSizeInHours); + stmt.execute(); + } catch (SQLException e) { + log.error("[{}] SQLException occurred during execution of {} with parameters {} and {}", functionName, startTs, partitionSizeInHours, e); + } + } + private void cleanupEvents(EventType eventType, long eventExpTime) { var partitionDuration = partitionConfiguration.getPartitionSizeInMs(eventType); List partitions = fetchPartitions(eventType); From 079c610ffc433dfca0b75136306741f12fd17e5c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 28 Jul 2022 13:28:59 +0300 Subject: [PATCH 11/13] Improvements to the upgrade script --- .../install/update/DefaultDataUpdateService.java | 6 ++++-- .../dao/sql/event/SqlEventCleanupRepository.java | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index d0d0322e0e..1eda4e027d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -164,10 +164,12 @@ public class DefaultDataUpdateService implements DataUpdateService { rateLimitsUpdater.updateEntities(); break; case "3.4.0": - if (System.getProperty("TB_EVENTS_MIGRATION", "false").equalsIgnoreCase("true")) { - log.info("Updating data from version 3.3.4 to 3.4.0 ..."); + String skipEventsMigration = System.getenv("TB_SKIP_EVENTS_MIGRATION"); + if (skipEventsMigration == null || skipEventsMigration.equalsIgnoreCase("false")) { + log.info("Updating data from version 3.4.0 to 3.4.1 ..."); eventService.migrateEvents(); } + break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index cdc8b5e348..9a4a7f13b5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -54,6 +54,17 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe public void migrateEvents(long regularEventTs, long debugEventTs) { callMigrateFunction("migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours()); callMigrateFunction("migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours()); + try (Connection connection = dataSource.getConnection(); + PreparedStatement dropFunction1 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_regular_events"); + PreparedStatement dropFunction2 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_debug_events"); + PreparedStatement dropTable = connection.prepareStatement("DROP TABLE IF EXISTS event")) { + dropFunction1.execute(); + dropFunction2.execute(); + dropTable.execute(); + } catch (SQLException e) { + log.error("SQLException occurred during drop of the `events` table", e); + throw new RuntimeException(e); + } } private void callMigrateFunction(String functionName, long startTs, int partitionSizeInHours) { @@ -63,7 +74,10 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe stmt.setInt(2, partitionSizeInHours); stmt.execute(); } catch (SQLException e) { - log.error("[{}] SQLException occurred during execution of {} with parameters {} and {}", functionName, startTs, partitionSizeInHours, e); + if (e.getMessage() == null || !e.getMessage().contains("relation \"event\" does not exist")) { + log.error("[{}] SQLException occurred during execution of {} with parameters {} and {}", functionName, startTs, partitionSizeInHours, e); + throw new RuntimeException(e); + } } } From 302f512be8d325dddff3ef12e098969ad74648d4 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 29 Jul 2022 15:43:13 +0300 Subject: [PATCH 12/13] Upgrade improvements --- .../main/data/upgrade/3.4.0/schema_update.sql | 81 +++++++++++-------- .../sql/event/SqlEventCleanupRepository.java | 41 ++++++++-- 2 files changed, 82 insertions(+), 40 deletions(-) diff --git a/application/src/main/data/upgrade/3.4.0/schema_update.sql b/application/src/main/data/upgrade/3.4.0/schema_update.sql index 2b4b5af42d..caac509d17 100644 --- a/application/src/main/data/upgrade/3.4.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.0/schema_update.sql @@ -88,9 +88,19 @@ CREATE INDEX IF NOT EXISTS idx_lc_event_main CREATE INDEX IF NOT EXISTS idx_error_event_main ON error_event (tenant_id ASC, entity_id ASC, ts DESC NULLS LAST) WITH (FILLFACTOR=95); +CREATE OR REPLACE FUNCTION to_safe_json(p_json text) RETURNS json +LANGUAGE plpgsql AS +$$ +BEGIN + return REPLACE(p_json, '\u0000', '' )::json; +EXCEPTION + WHEN OTHERS THEN + return '{}'::json; +END; +$$; -- Useful to migrate old events to the new table structure; -CREATE OR REPLACE PROCEDURE migrate_regular_events(IN start_ts_in_ms bigint, IN partition_size_in_hours int) +CREATE OR REPLACE PROCEDURE migrate_regular_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int) LANGUAGE plpgsql AS $$ DECLARE @@ -100,7 +110,7 @@ DECLARE BEGIN partition_size_in_ms = partition_size_in_hours * 3600 * 1000; - FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('STATS', 'LC_EVENT', 'ERROR') and ts > start_ts_in_ms + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('STATS', 'LC_EVENT', 'ERROR') and ts >= start_ts_in_ms and ts < end_ts_in_ms LOOP IF p.event_type = 'STATS' THEN table_name := 'stats_event'; @@ -121,9 +131,10 @@ BEGIN body::json ->> 'server', (body::json ->> 'messagesProcessed')::bigint, (body::json ->> 'errorsOccurred')::bigint - FROM event - WHERE ts > start_ts_in_ms - AND event_type = 'STATS' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'STATS' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event ON CONFLICT DO NOTHING; INSERT INTO lc_event @@ -135,9 +146,10 @@ BEGIN body::json ->> 'event', (body::json ->> 'success')::boolean, body::json ->> 'error' - FROM event - WHERE ts > start_ts_in_ms - AND event_type = 'LC_EVENT' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'LC_EVENT' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event ON CONFLICT DO NOTHING; INSERT INTO error_event @@ -148,16 +160,17 @@ BEGIN body::json ->> 'server', body::json ->> 'method', body::json ->> 'error' - FROM event - WHERE ts > start_ts_in_ms - AND event_type = 'ERROR' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'ERROR' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event ON CONFLICT DO NOTHING; END $$; -- Useful to migrate old debug events to the new table structure; -CREATE OR REPLACE PROCEDURE migrate_debug_events(IN start_ts_in_ms bigint, IN partition_size_in_hours int) +CREATE OR REPLACE PROCEDURE migrate_debug_events(IN start_ts_in_ms bigint, IN end_ts_in_ms bigint, IN partition_size_in_hours int) LANGUAGE plpgsql AS $$ DECLARE @@ -167,7 +180,7 @@ DECLARE BEGIN partition_size_in_ms = partition_size_in_hours * 3600 * 1000; - FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') and ts > start_ts_in_ms + FOR p IN SELECT DISTINCT event_type as event_type, (created_time - created_time % partition_size_in_ms) as partition_ts FROM event e WHERE e.event_type in ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') and ts >= start_ts_in_ms and ts < end_ts_in_ms LOOP IF p.event_type = 'DEBUG_RULE_NODE' THEN table_name := 'rule_node_debug_event'; @@ -183,20 +196,21 @@ BEGIN tenant_id, ts, entity_id, - body::json ->> 'server', - body::json ->> 'type', - (body::json ->> 'entityId')::uuid, - body::json ->> 'entityName', - (body::json ->> 'msgId')::uuid, - body::json ->> 'msgType', - body::json ->> 'dataType', - body::json ->> 'relationType', - body::json ->> 'data', - body::json ->> 'metadata', - body::json ->> 'error' - FROM event - WHERE ts > start_ts_in_ms - AND event_type = 'DEBUG_RULE_NODE' + body ->> 'server', + body ->> 'type', + (body ->> 'entityId')::uuid, + body ->> 'entityName', + (body ->> 'msgId')::uuid, + body ->> 'msgType', + body ->> 'dataType', + body ->> 'relationType', + body ->> 'data', + body ->> 'metadata', + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_NODE' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event ON CONFLICT DO NOTHING; INSERT INTO rule_chain_debug_event @@ -204,12 +218,13 @@ BEGIN tenant_id, ts, entity_id, - body::json ->> 'server', - body::json ->> 'message', - body::json ->> 'error' - FROM event - WHERE ts > start_ts_in_ms - AND event_type = 'DEBUG_RULE_CHAIN' + body ->> 'server', + body ->> 'message', + body ->> 'error' + FROM + (select id, tenant_id, ts, entity_id, to_safe_json(body) as body + FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'DEBUG_RULE_CHAIN' AND to_safe_json(body) ->> 'server' IS NOT NULL + ) safe_event ON CONFLICT DO NOTHING; END $$; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java index 9a4a7f13b5..53786c448b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/SqlEventCleanupRepository.java @@ -27,6 +27,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; @Slf4j @@ -52,11 +53,15 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe @Override public void migrateEvents(long regularEventTs, long debugEventTs) { - callMigrateFunction("migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours()); - callMigrateFunction("migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours()); + regularEventTs = Math.max(regularEventTs, 1480982400000L); + debugEventTs = Math.max(debugEventTs, 1480982400000L); + + callMigrateFunctionByPartitions("regular", "migrate_regular_events", regularEventTs, partitionConfiguration.getRegularPartitionSizeInHours()); + callMigrateFunctionByPartitions("debug", "migrate_debug_events", debugEventTs, partitionConfiguration.getDebugPartitionSizeInHours()); + try (Connection connection = dataSource.getConnection(); - PreparedStatement dropFunction1 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_regular_events"); - PreparedStatement dropFunction2 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_debug_events"); + PreparedStatement dropFunction1 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_regular_events(bigint, bigint, int)"); + PreparedStatement dropFunction2 = connection.prepareStatement("DROP PROCEDURE IF EXISTS migrate_debug_events(bigint, bigint, int)"); PreparedStatement dropTable = connection.prepareStatement("DROP TABLE IF EXISTS event")) { dropFunction1.execute(); dropFunction2.execute(); @@ -67,11 +72,33 @@ public class SqlEventCleanupRepository extends JpaAbstractDaoListeningExecutorSe } } - private void callMigrateFunction(String functionName, long startTs, int partitionSizeInHours) { + private void callMigrateFunctionByPartitions(String logTag, String functionName, long startTs, int partitionSizeInHours) { + long currentTs = System.currentTimeMillis(); + var regularPartitionStepInMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); + long numberOfPartitions = (currentTs - startTs) / regularPartitionStepInMs; + if (numberOfPartitions > 1000) { + log.error("Please adjust your {} events partitioning configuration. " + + "Configuration with partition size of {} hours and corresponding TTL will use {} (>1000) partitions which is not recommended!", + logTag, partitionSizeInHours, numberOfPartitions); + throw new RuntimeException("Please adjust your " + logTag + " events partitioning configuration. " + + "Configuration with partition size of " + partitionSizeInHours + " hours and corresponding TTL will use " + + +numberOfPartitions + " (>1000) partitions which is not recommended!"); + } + while (startTs < currentTs) { + var endTs = startTs + regularPartitionStepInMs; + log.info("Migrate {} events for time period: [{},{}]", logTag, startTs, endTs); + callMigrateFunction(functionName, startTs, startTs + regularPartitionStepInMs, partitionSizeInHours); + startTs = endTs; + } + log.info("Migrate {} events done.", logTag); + } + + private void callMigrateFunction(String functionName, long startTs, long endTs, int partitionSizeInHours) { try (Connection connection = dataSource.getConnection(); - PreparedStatement stmt = connection.prepareStatement("call " + functionName + "(?,?)")) { + PreparedStatement stmt = connection.prepareStatement("call " + functionName + "(?,?,?)")) { stmt.setLong(1, startTs); - stmt.setInt(2, partitionSizeInHours); + stmt.setLong(2, endTs); + stmt.setInt(3, partitionSizeInHours); stmt.execute(); } catch (SQLException e) { if (e.getMessage() == null || !e.getMessage().contains("relation \"event\" does not exist")) { From e602c98bc1584a3bb91937e315b335e2a140e24f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 29 Jul 2022 15:47:46 +0300 Subject: [PATCH 13/13] Upgrade improvements --- .../main/data/upgrade/3.4.0/schema_update.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/application/src/main/data/upgrade/3.4.0/schema_update.sql b/application/src/main/data/upgrade/3.4.0/schema_update.sql index caac509d17..12a86acebc 100644 --- a/application/src/main/data/upgrade/3.4.0/schema_update.sql +++ b/application/src/main/data/upgrade/3.4.0/schema_update.sql @@ -128,9 +128,9 @@ BEGIN tenant_id, ts, entity_id, - body::json ->> 'server', - (body::json ->> 'messagesProcessed')::bigint, - (body::json ->> 'errorsOccurred')::bigint + body ->> 'server', + (body ->> 'messagesProcessed')::bigint, + (body ->> 'errorsOccurred')::bigint FROM (select id, tenant_id, ts, entity_id, to_safe_json(body) as body FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'STATS' AND to_safe_json(body) ->> 'server' IS NOT NULL @@ -142,10 +142,10 @@ BEGIN tenant_id, ts, entity_id, - body::json ->> 'server', - body::json ->> 'event', - (body::json ->> 'success')::boolean, - body::json ->> 'error' + body ->> 'server', + body ->> 'event', + (body ->> 'success')::boolean, + body ->> 'error' FROM (select id, tenant_id, ts, entity_id, to_safe_json(body) as body FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'LC_EVENT' AND to_safe_json(body) ->> 'server' IS NOT NULL @@ -157,9 +157,9 @@ BEGIN tenant_id, ts, entity_id, - body::json ->> 'server', - body::json ->> 'method', - body::json ->> 'error' + body ->> 'server', + body ->> 'method', + body ->> 'error' FROM (select id, tenant_id, ts, entity_id, to_safe_json(body) as body FROM event WHERE ts >= start_ts_in_ms and ts < end_ts_in_ms AND event_type = 'ERROR' AND to_safe_json(body) ->> 'server' IS NOT NULL