added ttl for events

application/src/main/data/upgrade/2.4.3/schema_update_ttl.sql

@@ -123,3 +123,18 @@ BEGIN
         END LOOP;
 END
 $$;
+
+CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint)
+    LANGUAGE plpgsql AS
+$$
+    DECLARE
+        ttl_ts bigint;
+BEGIN
+    IF system_ttl > 0 THEN
+        ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint;
+        EXECUTE format(
+                'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted;
+    END IF;
+END
+$$;
+

application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@@ -221,6 +221,11 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
                             }
                         }
                     }
+                    try {
+                        long ts = System.currentTimeMillis();
+                        conn.createStatement().execute("ALTER TABLE event ADD COLUMN ts bigint DEFAULT " + ts + ";"); //tiNOSONAR, ignoring because method used to execute thingsboard database upgrade script
+                    } catch (Exception e) {
+                    }
                     log.info("Schema updated.");
                 }
                 break;

application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java

@@ -50,8 +50,7 @@ public class EventsCleanUpService extends AbstractCleanUpService {
 
     @Override
     protected void doCleanUp(Connection connection) {
-        log.info("ttl: [{}]", ttl);
-        log.info("ttlTaskExecutionEnabled: [{}]", ttlTaskExecutionEnabled);
-        // TODO: 4/15/20 Do a clean up.
+        long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", 0);");
+        log.info("Total events removed by TTL: [{}]", totalEventsRemoved);
     }
 }

dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@@ -32,6 +32,9 @@ public class ModelConstants {
     public static final String NULL_UUID_STR = UUIDConverter.fromTimeUUID(NULL_UUID);
     public static final TenantId SYSTEM_TENANT = new TenantId(ModelConstants.NULL_UUID);
 
+    // this is the difference between midnight October 15, 1582 UTC and midnight January 1, 1970 UTC as 100 nanosecond units
+    public static final long EPOCH_DIFF = 122192928000000000L;
+
     /**
      * Generic constants.
      */

dao/src/main/java/org/thingsboard/server/dao/model/sql/EventEntity.java

@@ -37,6 +37,9 @@ import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.Table;
 
+import java.util.UUID;
+
+import static org.thingsboard.server.dao.model.ModelConstants.EPOCH_DIFF;
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_BODY_PROPERTY;
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_COLUMN_FAMILY_NAME;
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_ID_PROPERTY;
@@ -44,6 +47,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.EVENT_ENTITY_TYPE_
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TENANT_ID_PROPERTY;
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_TYPE_PROPERTY;
 import static org.thingsboard.server.dao.model.ModelConstants.EVENT_UID_PROPERTY;
+import static org.thingsboard.server.dao.model.ModelConstants.TS_COLUMN;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -73,9 +77,13 @@ public class EventEntity  extends BaseSqlEntity<Event> implements BaseEntity<Eve
     @Column(name = EVENT_BODY_PROPERTY)
     private JsonNode body;
 
+    @Column(name = TS_COLUMN)
+    private long ts;
+
     public EventEntity(Event event) {
         if (event.getId() != null) {
             this.setId(event.getId().getId());
+            this.ts = getTs(event.getId().getId());
         }
         if (event.getTenantId() != null) {
             this.tenantId = toString(event.getTenantId().getId());
@@ -101,4 +109,8 @@ public class EventEntity  extends BaseSqlEntity<Event> implements BaseEntity<Eve
         event.setUid(eventUid);
         return event;
     }
+
+    private long getTs(UUID uuid) {
+        return (uuid.timestamp() - EPOCH_DIFF) / 1000;
+    }
 }

dao/src/main/resources/sql/schema-entities.sql

@@ -144,6 +144,7 @@ CREATE TABLE IF NOT EXISTS event (
     event_type varchar(255),
     event_uid varchar(255),
     tenant_id varchar(31),
+    ts bigint,
     CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid)
 );
 
@@ -249,3 +250,17 @@ CREATE TABLE IF NOT EXISTS entity_view (
     search_text varchar(255),
     additional_info varchar
 );
+
+CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl(IN system_ttl bigint, INOUT deleted bigint)
+    LANGUAGE plpgsql AS
+$$
+DECLARE
+    ttl_ts bigint;
+BEGIN
+    IF system_ttl > 0 THEN
+        ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - system_ttl::bigint * 1000)::bigint;
+        EXECUTE format(
+                'WITH deleted AS (DELETE FROM event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into deleted;
+    END IF;
+END
+$$;