From babe360507415e075f273fd176f705be404e3801 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 30 Jun 2021 20:40:38 +0300 Subject: [PATCH] events: performance improvements: UI fast response with latest events, ttl cleanup speedup (with schema update) --- .../upgrade/3.3.0/schema_update_event.sql | 90 +++++++++++++++++++ .../install/SqlDatabaseUpgradeService.java | 11 +++ .../service/ttl/EventsCleanUpService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 +- .../service/ttl/EventsCleanUpServiceTest.java | 36 ++++++++ 5 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 application/src/main/data/upgrade/3.3.0/schema_update_event.sql create mode 100644 application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java diff --git a/application/src/main/data/upgrade/3.3.0/schema_update_event.sql b/application/src/main/data/upgrade/3.3.0/schema_update_event.sql new file mode 100644 index 0000000000..6cbde51536 --- /dev/null +++ b/application/src/main/data/upgrade/3.3.0/schema_update_event.sql @@ -0,0 +1,90 @@ +-- +-- Copyright © 2016-2021 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- PROCEDURE: public.cleanup_events_by_ttl(bigint, bigint, bigint) + +DROP PROCEDURE IF EXISTS public.cleanup_events_by_ttl(bigint, bigint, bigint); + +CREATE OR REPLACE PROCEDURE public.cleanup_events_by_ttl( + ttl bigint, + debug_ttl bigint, + INOUT deleted bigint) +LANGUAGE 'plpgsql' +AS $BODY$ +DECLARE + ttl_ts bigint; + debug_ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + + DELETE FROM event + WHERE ts < ttl_ts + AND NOT event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION'); + + GET DIAGNOSTICS ttl_deleted_count = ROW_COUNT; + END IF; + + IF debug_ttl > 0 THEN + debug_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - debug_ttl::bigint * 1000)::bigint; + + DELETE FROM event + WHERE ts < debug_ttl_ts + AND event_type IN ('DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN', 'DEBUG_CONVERTER', 'DEBUG_INTEGRATION'); + + GET DIAGNOSTICS debug_ttl_deleted_count = ROW_COUNT; + END IF; + + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$BODY$; + + +-- Index: idx_event_ts + +DROP INDEX IF EXISTS public.idx_event_ts; + +-- Hint: add CONCURRENTLY to CREATE INDEX query in case of more then 1 million records or during live update +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_ts +CREATE INDEX IF NOT EXISTS idx_event_ts + ON public.event USING btree + (ts DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_ts + IS 'This index helps to delete events by TTL using timestamp'; + + +-- Index: idx_event_tenant_entity_type_entity_event_type_created_time_des + +DROP INDEX IF EXISTS public.idx_event_tenant_entity_type_entity_event_type_created_time_des; + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des +CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des + ON public.event USING btree + (tenant_id ASC NULLS LAST, entity_type ASC NULLS LAST, entity_id ASC NULLS LAST, event_type ASC NULLS LAST, created_time DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des + IS 'This index helps to open latest events on UI fast'; + +-- Index: idx_event_type_entity_id +-- Description: replaced with more suitable idx_event_tenant_entity_type_entity_event_type_created_time_des +DROP INDEX IF EXISTS public.idx_event_type_entity_id; \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 470f39f401..cb0b83abb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -465,6 +465,17 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.0": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + log.info("Updating indexes and procedure for event table..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.0", "schema_update_event.sql"); + loadSql(schemaUpdateFile, conn); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java index a51910b7ed..e2d6842241 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/EventsCleanUpService.java @@ -45,7 +45,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { this.eventService = eventService; } - @Scheduled(initialDelayString = "${sql.ttl.events.execution_interval_ms}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") + @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.events.execution_interval_ms})}", fixedDelayString = "${sql.ttl.events.execution_interval_ms}") public void cleanUp() { if (ttlTaskExecutionEnabled && isSystemTenantPartitionMine()) { eventService.cleanupEvents(ttl, debugTtl); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 05f3f1e825..377b2b4461 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -266,7 +266,7 @@ sql: ts_key_value_ttl: "${SQL_TTL_TS_TS_KEY_VALUE_TTL:0}" # Number of seconds events: enabled: "${SQL_TTL_EVENTS_ENABLED:true}" - execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:86400000}" # Number of milliseconds. The current value corresponds to one day + execution_interval_ms: "${SQL_TTL_EVENTS_EXECUTION_INTERVAL:2220000}" # Number of milliseconds (max random initial delay and fixed period). # 37minutes to avoid common interval spikes events_ttl: "${SQL_TTL_EVENTS_EVENTS_TTL:0}" # Number of seconds debug_events_ttl: "${SQL_TTL_EVENTS_DEBUG_EVENTS_TTL:604800}" # Number of seconds. The current value corresponds to one week edge_events: diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java new file mode 100644 index 0000000000..b2eb9f39f1 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/EventsCleanUpServiceTest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +@Slf4j +public class EventsCleanUpServiceTest { + + @Test + public void givenInterval_whenRandomDelay_ThenDelayInInterval() { + final long executionIntervalMs = 2220000; //37min + final long randomDelay = org.apache.commons.lang3.RandomUtils.nextLong(0, executionIntervalMs); //same as @Scheduled(initialDelayString = ... + log.info("randomDelay {}", randomDelay); + assertThat(randomDelay, greaterThanOrEqualTo(0L)); + assertThat(randomDelay, lessThanOrEqualTo(executionIntervalMs)); + } +}