diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 89c79fcb8b..2c202e71b2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -26,26 +27,27 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -73,7 +75,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; @@ -640,6 +641,12 @@ public abstract class BaseController { case ALARM_CLEAR: msgType = DataConstants.ALARM_CLEAR; break; + case ASSIGNED_FROM_TENANT: + msgType = DataConstants.ENTITY_ASSIGNED_FROM_TENANT; + break; + case ASSIGNED_TO_TENANT: + msgType = DataConstants.ENTITY_ASSIGNED_TO_TENANT; + break; } if (!StringUtils.isEmpty(msgType)) { try { @@ -659,6 +666,16 @@ public abstract class BaseController { String strCustomerName = extractParameter(String.class, 2, additionalInfo); metaData.putValue("unassignedCustomerId", strCustomerId); metaData.putValue("unassignedCustomerName", strCustomerName); + } else if (actionType == ActionType.ASSIGNED_FROM_TENANT) { + String strTenantId = extractParameter(String.class, 0, additionalInfo); + String strTenantName = extractParameter(String.class, 1, additionalInfo); + metaData.putValue("assignedFromTenantId", strTenantId); + metaData.putValue("assignedFromTenantName", strTenantName); + } else if (actionType == ActionType.ASSIGNED_TO_TENANT) { + String strTenantId = extractParameter(String.class, 0, additionalInfo); + String strTenantName = extractParameter(String.class, 1, additionalInfo); + metaData.putValue("assignedToTenantId", strTenantId); + metaData.putValue("assignedToTenantName", strTenantName); } ObjectNode entityNode; if (entity != null) { @@ -722,5 +739,13 @@ public abstract class BaseController { return result; } + protected String entityToStr(E entity) { + try { + return json.writeValueAsString(json.valueToTree(entity)); + } catch (JsonProcessingException e) { + log.warn("[{}] Failed to convert entity to string!", entity, e); + } + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index d96e0e5dc2..d0a662d2ec 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -40,8 +40,10 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.device.DeviceSearchQuery; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -49,6 +51,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -71,6 +76,7 @@ public class DeviceController extends BaseController { private static final String DEVICE_ID = "deviceId"; private static final String DEVICE_NAME = "deviceName"; + private static final String TENANT_ID = "tenantId"; @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/device/{deviceId}", method = RequestMethod.GET) @@ -547,4 +553,54 @@ public class DeviceController extends BaseController { } return DataConstants.DEFAULT_SECRET_KEY; } + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/{tenantId}/device/{deviceId}", method = RequestMethod.POST) + @ResponseBody + public Device assignDeviceToTenant(@PathVariable(TENANT_ID) String strTenantId, + @PathVariable(DEVICE_ID) String strDeviceId) throws ThingsboardException { + checkParameter(TENANT_ID, strTenantId); + checkParameter(DEVICE_ID, strDeviceId); + try { + DeviceId deviceId = new DeviceId(toUUID(strDeviceId)); + Device device = checkDeviceId(deviceId, Operation.ASSIGN_TO_TENANT); + + TenantId newTenantId = new TenantId(toUUID(strTenantId)); + Tenant newTenant = tenantService.findTenantById(newTenantId); + if (newTenant == null) { + throw new ThingsboardException("Could not find the specified Tenant!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + + Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); + + logEntityAction(getCurrentUser(), deviceId, assignedDevice, + assignedDevice.getCustomerId(), + ActionType.ASSIGNED_TO_TENANT, null, strTenantId, newTenant.getName()); + + Tenant currentTenant = tenantService.findTenantById(getTenantId()); + pushAssignedFromNotification(currentTenant, newTenantId, assignedDevice); + + return assignedDevice; + } catch (Exception e) { + logEntityAction(getCurrentUser(), emptyId(EntityType.DEVICE), null, + null, + ActionType.ASSIGNED_TO_TENANT, e, strTenantId); + throw handleException(e); + } + } + + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { + String data = entityToStr(assignedDevice); + if (data != null) { + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data); + tbClusterService.pushMsgToRuleEngine(newTenantId, assignedDevice.getId(), tbMsg, null); + } + } + + private TbMsgMetaData getMetaDataForAssignedFrom(Tenant tenant) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("assignedFromTenantId", tenant.getId().getId().toString()); + metaData.putValue("assignedFromTenantName", tenant.getName()); + return metaData; + } } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 1cc4146ed7..7d80b115d5 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -28,7 +28,9 @@ import org.thingsboard.server.service.install.DatabaseTsUpgradeService; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import org.thingsboard.server.service.install.SystemDataLoaderService; import org.thingsboard.server.service.install.TsDatabaseSchemaService; +import org.thingsboard.server.service.install.TsLatestDatabaseSchemaService; import org.thingsboard.server.service.install.migrate.EntitiesMigrateService; +import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.DataUpdateService; @Service @@ -51,6 +53,9 @@ public class ThingsboardInstallService { @Autowired private TsDatabaseSchemaService tsDatabaseSchemaService; + @Autowired(required = false) + private TsLatestDatabaseSchemaService tsLatestDatabaseSchemaService; + @Autowired private DatabaseEntitiesUpgradeService databaseEntitiesUpgradeService; @@ -72,6 +77,9 @@ public class ThingsboardInstallService { @Autowired(required = false) private EntitiesMigrateService entitiesMigrateService; + @Autowired(required = false) + private TsLatestMigrateService latestMigrateService; + public void performInstall() { try { if (isUpgrade) { @@ -82,6 +90,10 @@ public class ThingsboardInstallService { entitiesMigrateService.migrate(); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); + } else if ("3.0.1-cassandra".equals(upgradeFromVersion)) { + log.info("Migrating ThingsBoard latest timeseries data from cassandra to SQL database ..."); + latestMigrateService.migrate(); + log.info("Updating system data..."); } else { switch (upgradeFromVersion) { case "1.2.3": //NOSONAR, Need to execute gradual upgrade starting from upgradeFromVersion @@ -181,6 +193,10 @@ public class ThingsboardInstallService { tsDatabaseSchemaService.createDatabaseSchema(); + if (tsLatestDatabaseSchemaService != null) { + tsLatestDatabaseSchemaService.createDatabaseSchema(); + } + log.info("Loading system data..."); componentDiscoveryService.discoverComponents(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java new file mode 100644 index 0000000000..f2df8a50da --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsLatestDatabaseSchemaService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.dao.util.NoSqlTsLatestDao; + +@Service +@NoSqlTsLatestDao +@Profile("install") +public class CassandraTsLatestDatabaseSchemaService extends CassandraAbstractDatabaseSchemaService + implements TsLatestDatabaseSchemaService { + public CassandraTsLatestDatabaseSchemaService() { + super("schema-ts-latest.cql"); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java index 0b3232903b..8124356bba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/HsqlEntityDatabaseSchemaService.java @@ -18,11 +18,9 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Service @HsqlDao -@SqlDao @Profile("install") public class HsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements EntityDatabaseSchemaService { diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java index 11da8b306a..364457f11e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java @@ -18,10 +18,8 @@ package org.thingsboard.server.service.install; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Service -@SqlDao @PsqlDao @Profile("install") public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index e244c307ff..e21472ee12 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.install.sql.SqlDbHelper; import java.nio.charset.Charset; @@ -58,7 +57,6 @@ import static org.thingsboard.server.service.install.DatabaseHelper.TYPE; @Service @Profile("install") @Slf4j -@SqlDao public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService { private static final String SCHEMA_UPDATE_SQL = "schema_update.sql"; diff --git a/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java b/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java new file mode 100644 index 0000000000..070cc2060e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/TsLatestDatabaseSchemaService.java @@ -0,0 +1,19 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +public interface TsLatestDatabaseSchemaService extends DatabaseSchemaService { +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java index a4b4336d61..77459604ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraEntitiesToSqlMigrateService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.UUIDConverter; import org.thingsboard.server.dao.cassandra.CassandraCluster; import org.thingsboard.server.dao.util.NoSqlAnyDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.install.EntityDatabaseSchemaService; import java.sql.Connection; @@ -42,7 +41,6 @@ import static org.thingsboard.server.service.install.migrate.CassandraToSqlColum @Service @Profile("install") -@SqlDao @NoSqlAnyDao @Slf4j public class CassandraEntitiesToSqlMigrateService implements EntitiesMigrateService { diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java index dd67a6b055..b18f48ec85 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraToSqlTable.java @@ -128,7 +128,7 @@ public class CassandraToSqlTable { return this.validateColumnData(data); } - private CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { + protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { for (int i=0;i batchData, Connection conn) throws SQLException { + protected void batchInsert(List batchData, Connection conn) throws SQLException { boolean retry = false; for (CassandraToSqlColumnData[] data : batchData) { for (CassandraToSqlColumn column: this.columns) { @@ -269,7 +269,7 @@ public class CassandraToSqlTable { return Optional.empty(); } - private Statement createCassandraSelectStatement() { + protected Statement createCassandraSelectStatement() { StringBuilder selectStatementBuilder = new StringBuilder(); selectStatementBuilder.append("SELECT "); for (CassandraToSqlColumn column : columns) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java new file mode 100644 index 0000000000..ab22e3748c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/CassandraTsLatestToSqlMigrateService.java @@ -0,0 +1,205 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.migrate; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.UUIDConverter; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; +import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; +import org.thingsboard.server.dao.util.NoSqlAnyDao; +import org.thingsboard.server.service.install.EntityDatabaseSchemaService; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.bigintColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.booleanColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.doubleColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.idColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.jsonColumn; +import static org.thingsboard.server.service.install.migrate.CassandraToSqlColumn.stringColumn; + +@Service +@Profile("install") +@NoSqlAnyDao +@Slf4j +public class CassandraTsLatestToSqlMigrateService implements TsLatestMigrateService { + + @Autowired + private EntityDatabaseSchemaService entityDatabaseSchemaService; + + @Autowired + private InsertLatestTsRepository insertLatestTsRepository; + + @Autowired + protected CassandraCluster cluster; + + @Autowired + protected TsKvDictionaryRepository dictionaryRepository; + + @Value("${spring.datasource.url}") + protected String dbUrl; + + @Value("${spring.datasource.username}") + protected String dbUserName; + + @Value("${spring.datasource.password}") + protected String dbPassword; + + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + + protected static final ReentrantLock tsCreationLock = new ReentrantLock(); + + @Override + public void migrate() throws Exception { + log.info("Performing migration of latest timeseries data from cassandra to SQL database ..."); + entityDatabaseSchemaService.createDatabaseSchema(false); + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + conn.setAutoCommit(false); + for (CassandraToSqlTable table : tables) { + table.migrateToSql(cluster.getSession(), conn); + } + } catch (Exception e) { + log.error("Unexpected error during ThingsBoard entities data migration!", e); + throw e; + } + entityDatabaseSchemaService.createDatabaseIndexes(); + } + + private List tables = Arrays.asList( + new CassandraToSqlTable("ts_kv_latest_cf", + idColumn("entity_id"), + stringColumn("key"), + bigintColumn("ts"), + booleanColumn("bool_v"), + stringColumn("str_v"), + bigintColumn("long_v"), + doubleColumn("dbl_v"), + jsonColumn("json_v")) { + + @Override + protected void batchInsert(List batchData, Connection conn) { + insertLatestTsRepository + .saveOrUpdate(batchData.stream().map(data -> getTsKvLatestEntity(data)).collect(Collectors.toList())); + } + + @Override + protected CassandraToSqlColumnData[] validateColumnData(CassandraToSqlColumnData[] data) { + return data; + } + }); + + private TsKvLatestEntity getTsKvLatestEntity(CassandraToSqlColumnData[] data) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(UUIDConverter.fromString(data[0].getValue())); + latestEntity.setKey(getOrSaveKeyId(data[1].getValue())); + latestEntity.setTs(Long.parseLong(data[2].getValue())); + + String strV = data[4].getValue(); + if (strV != null) { + latestEntity.setStrValue(strV); + } else { + Long longV = null; + try { + longV = Long.parseLong(data[5].getValue()); + } catch (Exception e) { + } + if (longV != null) { + latestEntity.setLongValue(longV); + } else { + Double doubleV = null; + try { + doubleV = Double.parseDouble(data[6].getValue()); + } catch (Exception e) { + } + if (doubleV != null) { + latestEntity.setDoubleValue(doubleV); + } else { + + String jsonV = data[7].getValue(); + if (StringUtils.isNoneEmpty(jsonV)) { + latestEntity.setJsonValue(jsonV); + } else { + Boolean boolV = null; + try { + boolV = Boolean.parseBoolean(data[3].getValue()); + } catch (Exception e) { + } + if (boolV != null) { + latestEntity.setBooleanValue(boolV); + } else { + log.warn("All values in key-value row are nullable "); + } + } + } + } + } + return latestEntity; + } + + protected Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java b/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java new file mode 100644 index 0000000000..491cac2c99 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/migrate/TsLatestMigrateService.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.migrate; + +public interface TsLatestMigrateService { + + void migrate() throws Exception; +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 5fe6e0af1a..6e1eca4a7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.util.mapping.JacksonUtil; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.LocalSubscriptionServiceMsgProto; @@ -61,7 +60,6 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 0c4f62585d..2334c01102 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -18,6 +18,6 @@ package org.thingsboard.server.service.security.permission; public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, - READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES + READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, ASSIGN_TO_TENANT } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index 5b094c5c0e..a608ca257b 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.service.ttl.AbstractCleanUpService; import java.sql.Connection; @@ -28,7 +27,6 @@ import java.sql.DriverManager; import java.sql.SQLException; @PsqlDao -@SqlDao @Slf4j @Service public class EventsCleanUpService extends AbstractCleanUpService { diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index d6b08195b2..fb09a7eab4 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -19,11 +19,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.util.PsqlTsDao; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsDao; import java.sql.Connection; -@PsqlTsDao +@SqlTsDao +@PsqlDao @Service @Slf4j public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { @@ -33,9 +35,9 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi @Override protected void doCleanUp(Connection connection) { - long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); - log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); - long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);"); - log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); + long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); + log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); + long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID + "'," + systemTtl + ", 0);"); + log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); } } \ No newline at end of file diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a482f4a219..bc6eebcf23 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -178,8 +178,8 @@ database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by single API call to fetch telemetry records ts: type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) - latest_ts: - type: "${DATABASE_TS_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) + ts_latest: + type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) # note: timescale works only with postgreSQL database for DATABASE_ENTITIES_TYPE. diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 431ff385fc..f57b5a1dfd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -15,74 +15,17 @@ */ package org.thingsboard.server.controller; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Header; -import io.jsonwebtoken.Jwt; -import io.jsonwebtoken.Jwts; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.hamcrest.Matcher; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootContextLoader; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.http.HttpHeaders; -import org.springframework.http.MediaType; -import org.springframework.http.converter.HttpMessageConverter; -import org.springframework.http.converter.StringHttpMessageConverter; -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; -import org.springframework.mock.http.MockHttpInputMessage; -import org.springframework.mock.http.MockHttpOutputMessage; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.test.context.web.WebAppConfiguration; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.MvcResult; -import org.springframework.test.web.servlet.ResultActions; -import org.springframework.test.web.servlet.ResultMatcher; -import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.context.WebApplicationContext; -import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.page.TimePageLink; -import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.config.ThingsboardSecurityConfiguration; -import org.thingsboard.server.service.mail.TestMailService; -import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; -import org.thingsboard.server.service.security.auth.rest.LoginRequest; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; -import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; @ActiveProfiles("test") @RunWith(SpringRunner.class) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index afa08c05db..992b39c472 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -218,6 +218,7 @@ public abstract class AbstractWebTest { } private Tenant savedDifferentTenant; + protected void loginDifferentTenant() throws Exception { loginSysAdmin(); Tenant tenant = new Tenant(); @@ -313,6 +314,10 @@ public abstract class AbstractWebTest { return readResponse(doGet(urlTemplate, urlVariables).andExpect(status().isOk()), responseClass); } + protected T doGet(String urlTemplate, Class responseClass, ResultMatcher resultMatcher, Object... urlVariables) throws Exception { + return readResponse(doGet(urlTemplate, urlVariables).andExpect(resultMatcher), responseClass); + } + protected T doGetAsync(String urlTemplate, Class responseClass, Object... urlVariables) throws Exception { return readResponse(doGetAsync(urlTemplate, urlVariables).andExpect(status().isOk()), responseClass); } @@ -352,9 +357,9 @@ public abstract class AbstractWebTest { return readResponse(doGet(urlTemplate, vars).andExpect(status().isOk()), responseType); } - protected T doGetTypedWithTimePageLink(String urlTemplate, TypeReference responseType, - TimePageLink pageLink, - Object... urlVariables) throws Exception { + protected T doGetTypedWithTimePageLink(String urlTemplate, TypeReference responseType, + TimePageLink pageLink, + Object... urlVariables) throws Exception { List pageLinkVariables = new ArrayList<>(); urlTemplate += "pageSize={pageSize}&page={page}"; pageLinkVariables.add(pageLink.getPageSize()); @@ -395,11 +400,11 @@ public abstract class AbstractWebTest { return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseClass); } - protected R doPostWithResponse(String urlTemplate, T content, Class responseClass, String... params) throws Exception { + protected R doPostWithResponse(String urlTemplate, T content, Class responseClass, String... params) throws Exception { return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseClass); } - protected R doPostWithTypedResponse(String urlTemplate, T content, TypeReference responseType, String... params) throws Exception { + protected R doPostWithTypedResponse(String urlTemplate, T content, TypeReference responseType, String... params) throws Exception { return readResponse(doPost(urlTemplate, content, params).andExpect(status().isOk()), responseType); } @@ -430,7 +435,7 @@ public abstract class AbstractWebTest { return mockMvc.perform(postRequest); } - protected ResultActions doPostAsync(String urlTemplate, T content, Long timeout, String... params) throws Exception { + protected ResultActions doPostAsync(String urlTemplate, T content, Long timeout, String... params) throws Exception { MockHttpServletRequestBuilder postRequest = post(urlTemplate, params); setJwtToken(postRequest); String json = json(content); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index c1ad24bb04..f87dc5de6d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -15,74 +15,79 @@ */ package org.thingsboard.server.controller; -import static org.hamcrest.Matchers.containsString; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.RandomStringUtils; -import org.thingsboard.server.common.data.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceCredentialsId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.model.ModelConstants; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import com.fasterxml.jackson.core.type.TypeReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public abstract class BaseDeviceControllerTest extends AbstractControllerTest { - + private IdComparator idComparator = new IdComparator<>(); - + private Tenant savedTenant; private User tenantAdmin; - + @Before public void beforeTest() throws Exception { loginSysAdmin(); - + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); savedTenant = doPost("/api/tenant", tenant, Tenant.class); Assert.assertNotNull(savedTenant); - + tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); - + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); } - + @After public void afterTest() throws Exception { loginSysAdmin(); - - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + .andExpect(status().isOk()); } - + @Test public void testSaveDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + Assert.assertNotNull(savedDevice); Assert.assertNotNull(savedDevice.getId()); Assert.assertTrue(savedDevice.getCreatedTime() > 0); @@ -90,9 +95,9 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertNotNull(savedDevice.getCustomerId()); Assert.assertEquals(NULL_UUID, savedDevice.getCustomerId().getId()); Assert.assertEquals(device.getName(), savedDevice.getName()); - - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertNotNull(deviceCredentials); Assert.assertNotNull(deviceCredentials.getId()); @@ -100,10 +105,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(DeviceCredentialsType.ACCESS_TOKEN, deviceCredentials.getCredentialsType()); Assert.assertNotNull(deviceCredentials.getCredentialsId()); Assert.assertEquals(20, deviceCredentials.getCredentialsId().length()); - + savedDevice.setName("My new device"); doPost("/api/device", savedDevice, Device.class); - + Device foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(foundDevice.getName(), savedDevice.getName()); } @@ -115,10 +120,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); loginDifferentTenant(); - doPost("/api/device", savedDevice, Device.class, status().isForbidden()); + doPost("/api/device", savedDevice, Device.class, status().isNotFound()); deleteDifferentTenant(); } - + @Test public void testFindDeviceById() throws Exception { Device device = new Device(); @@ -133,26 +138,27 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { @Test public void testFindDeviceTypesByTenantId() throws Exception { List devices = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Device device = new Device(); - device.setName("My device B"+i); + device.setName("My device B" + i); device.setType("typeB"); devices.add(doPost("/api/device", device, Device.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Device device = new Device(); - device.setName("My device C"+i); + device.setName("My device C" + i); device.setType("typeC"); devices.add(doPost("/api/device", device, Device.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Device device = new Device(); - device.setName("My device A"+i); + device.setName("My device A" + i); device.setType("typeA"); devices.add(doPost("/api/device", device, Device.class)); } List deviceTypes = doGetTyped("/api/device/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(deviceTypes); Assert.assertEquals(3, deviceTypes.size()); @@ -160,19 +166,19 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals("typeB", deviceTypes.get(1).getType()); Assert.assertEquals("typeC", deviceTypes.get(2).getType()); } - + @Test public void testDeleteDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - - doDelete("/api/device/"+savedDevice.getId().getId().toString()) - .andExpect(status().isOk()); - doGet("/api/device/"+savedDevice.getId().getId().toString()) - .andExpect(status().isNotFound()); + doDelete("/api/device/" + savedDevice.getId().getId().toString()) + .andExpect(status().isOk()); + + doGet("/api/device/" + savedDevice.getId().getId().toString()) + .andExpect(status().isNotFound()); } @Test @@ -189,52 +195,51 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Device device = new Device(); device.setType("default"); doPost("/api/device", device) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device name should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device name should be specified"))); } - + @Test public void testAssignUnassignDeviceToCustomer() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + Customer customer = new Customer(); customer.setTitle("My customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - - Device assignedDevice = doPost("/api/customer/" + savedCustomer.getId().getId().toString() + + Device assignedDevice = doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(savedCustomer.getId(), assignedDevice.getCustomerId()); - + Device foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(savedCustomer.getId(), foundDevice.getCustomerId()); - Device unassignedDevice = + Device unassignedDevice = doDelete("/api/customer/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(ModelConstants.NULL_UUID, unassignedDevice.getCustomerId().getId()); - + foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(ModelConstants.NULL_UUID, foundDevice.getCustomerId().getId()); } - + @Test public void testAssignDeviceToNonExistentCustomer() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - doPost("/api/customer/" + Uuids.timeBased().toString() + "/device/" + savedDevice.getId().getId().toString()) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()); } - + @Test public void testAssignDeviceToCustomerFromDifferentTenant() throws Exception { loginSysAdmin(); - + Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); @@ -246,103 +251,103 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { tenantAdmin2.setEmail("tenant3@thingsboard.org"); tenantAdmin2.setFirstName("Joe"); tenantAdmin2.setLastName("Downs"); - + tenantAdmin2 = createUserAndLogin(tenantAdmin2, "testPassword1"); - + Customer customer = new Customer(); customer.setTitle("Different customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); login(tenantAdmin.getEmail(), "testPassword1"); - + Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString()) - .andExpect(status().isForbidden()); - + .andExpect(status().isForbidden()); + loginSysAdmin(); - - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) + .andExpect(status().isOk()); } - + @Test public void testFindDeviceCredentialsByDeviceId() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertEquals(savedDevice.getId(), deviceCredentials.getDeviceId()); } - + @Test public void testSaveDeviceCredentials() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertEquals(savedDevice.getId(), deviceCredentials.getDeviceId()); deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); deviceCredentials.setCredentialsId("access_token"); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isOk()); - - DeviceCredentials foundDeviceCredentials = + .andExpect(status().isOk()); + + DeviceCredentials foundDeviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); - + Assert.assertEquals(deviceCredentials, foundDeviceCredentials); } - + @Test public void testSaveDeviceCredentialsWithEmptyDevice() throws Exception { DeviceCredentials deviceCredentials = new DeviceCredentials(); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()); + .andExpect(status().isBadRequest()); } - + @Test public void testSaveDeviceCredentialsWithEmptyCredentialsType() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setCredentialsType(null); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device credentials type should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device credentials type should be specified"))); } - + @Test public void testSaveDeviceCredentialsWithEmptyCredentialsId() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setCredentialsId(null); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device credentials id should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device credentials id should be specified"))); } - + @Test public void testSaveNonExistentDeviceCredentials() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); DeviceCredentials newDeviceCredentials = new DeviceCredentials(new DeviceCredentialsId(Uuids.timeBased())); newDeviceCredentials.setCreatedTime(deviceCredentials.getCreatedTime()); @@ -350,29 +355,29 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { newDeviceCredentials.setCredentialsType(deviceCredentials.getCredentialsType()); newDeviceCredentials.setCredentialsId(deviceCredentials.getCredentialsId()); doPost("/api/device/credentials", newDeviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Unable to update non-existent device credentials"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Unable to update non-existent device credentials"))); } - + @Test public void testSaveDeviceCredentialsWithNonExistentDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setDeviceId(new DeviceId(Uuids.timeBased())); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()); } @Test public void testFindTenantDevices() throws Exception { List devices = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Device device = new Device(); - device.setName("Device"+i); + device.setName("Device" + i); device.setType("default"); devices.add(doPost("/api/device", device, Device.class)); } @@ -380,28 +385,29 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { PageLink pageLink = new PageLink(23); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", + pageData = doGetTypedWithPageLink("/api/tenant/devices?", new TypeReference>(){}, pageLink); + loadedDevices.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devices, idComparator); Collections.sort(loadedDevices, idComparator); - + Assert.assertEquals(devices, loadedDevices); } - + @Test public void testFindTenantDevicesByName() throws Exception { String title1 = "Device title 1"; List devicesTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); @@ -409,37 +415,37 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { } String title2 = "Device title 2"; List devicesTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); devicesTitle2.add(doPost("/api/device", device, Device.class)); } - + List loadedDevicesTitle1 = new ArrayList<>(); PageLink pageLink = new PageLink(15, 0, title1); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", + pageData = doGetTypedWithPageLink("/api/tenant/devices?", new TypeReference>(){}, pageLink); loadedDevicesTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devicesTitle1, idComparator); Collections.sort(loadedDevicesTitle1, idComparator); - + Assert.assertEquals(devicesTitle1, loadedDevicesTitle1); - + List loadedDevicesTitle2 = new ArrayList<>(); pageLink = new PageLink(4, 0, title2); do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", + pageData = doGetTypedWithPageLink("/api/tenant/devices?", new TypeReference>(){}, pageLink); loadedDevicesTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { @@ -449,25 +455,23 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Collections.sort(devicesTitle2, idComparator); Collections.sort(loadedDevicesTitle2, idComparator); - + Assert.assertEquals(devicesTitle2, loadedDevicesTitle2); - + for (Device device : loadedDevicesTitle1) { - doDelete("/api/device/"+device.getId().getId().toString()) - .andExpect(status().isOk()); + doDelete("/api/device/" + device.getId().getId().toString()) + .andExpect(status().isOk()); } - pageLink = new PageLink(4, 0, title1); pageData = doGetTypedWithPageLink("/api/tenant/devices?", new TypeReference>(){}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); - + for (Device device : loadedDevicesTitle2) { - doDelete("/api/device/"+device.getId().getId().toString()) - .andExpect(status().isOk()); + doDelete("/api/device/" + device.getId().getId().toString()) + .andExpect(status().isOk()); } - pageLink = new PageLink(4, 0, title2); pageData = doGetTypedWithPageLink("/api/tenant/devices?", new TypeReference>(){}, pageLink); @@ -480,10 +484,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; String type1 = "typeA"; List devicesType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type1); @@ -492,10 +496,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title2 = "Device title 2"; String type2 = "typeB"; List devicesType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type2); @@ -536,7 +540,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(devicesType2, loadedDevicesType2); for (Device device : loadedDevicesType1) { - doDelete("/api/device/"+device.getId().getId().toString()) + doDelete("/api/device/" + device.getId().getId().toString()) .andExpect(status().isOk()); } @@ -547,7 +551,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); for (Device device : loadedDevicesType2) { - doDelete("/api/device/"+device.getId().getId().toString()) + doDelete("/api/device/" + device.getId().getId().toString()) .andExpect(status().isOk()); } @@ -557,42 +561,42 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } - + @Test public void testFindCustomerDevices() throws Exception { Customer customer = new Customer(); customer.setTitle("Test customer"); customer = doPost("/api/customer", customer, Customer.class); CustomerId customerId = customer.getId(); - + List devices = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Device device = new Device(); - device.setName("Device"+i); + device.setName("Device" + i); device.setType("default"); device = doPost("/api/device", device, Device.class); - devices.add(doPost("/api/customer/" + customerId.getId().toString() - + "/device/" + device.getId().getId().toString(), Device.class)); + devices.add(doPost("/api/customer/" + customerId.getId().toString() + + "/device/" + device.getId().getId().toString(), Device.class)); } - + List loadedDevices = new ArrayList<>(); PageLink pageLink = new PageLink(23); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", new TypeReference>(){}, pageLink); loadedDevices.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devices, idComparator); Collections.sort(loadedDevices, idComparator); - + Assert.assertEquals(devices, loadedDevices); } - + @Test public void testFindCustomerDevicesByName() throws Exception { Customer customer = new Customer(); @@ -602,52 +606,52 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; List devicesTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); device = doPost("/api/device", device, Device.class); - devicesTitle1.add(doPost("/api/customer/" + customerId.getId().toString() + devicesTitle1.add(doPost("/api/customer/" + customerId.getId().toString() + "/device/" + device.getId().getId().toString(), Device.class)); } String title2 = "Device title 2"; List devicesTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); device = doPost("/api/device", device, Device.class); - devicesTitle2.add(doPost("/api/customer/" + customerId.getId().toString() + devicesTitle2.add(doPost("/api/customer/" + customerId.getId().toString() + "/device/" + device.getId().getId().toString(), Device.class)); } - + List loadedDevicesTitle1 = new ArrayList<>(); PageLink pageLink = new PageLink(15, 0, title1); PageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", new TypeReference>(){}, pageLink); loadedDevicesTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devicesTitle1, idComparator); Collections.sort(loadedDevicesTitle1, idComparator); - + Assert.assertEquals(devicesTitle1, loadedDevicesTitle1); - + List loadedDevicesTitle2 = new ArrayList<>(); pageLink = new PageLink(4, 0, title2); do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", new TypeReference>(){}, pageLink); loadedDevicesTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { @@ -657,25 +661,23 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Collections.sort(devicesTitle2, idComparator); Collections.sort(loadedDevicesTitle2, idComparator); - + Assert.assertEquals(devicesTitle2, loadedDevicesTitle2); - + for (Device device : loadedDevicesTitle1) { doDelete("/api/customer/device/" + device.getId().getId().toString()) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } - pageLink = new PageLink(4, 0, title1); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", new TypeReference>(){}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); - + for (Device device : loadedDevicesTitle2) { doDelete("/api/customer/device/" + device.getId().getId().toString()) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } - pageLink = new PageLink(4, 0, title2); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", new TypeReference>(){}, pageLink); @@ -693,10 +695,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; String type1 = "typeC"; List devicesType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type1); @@ -707,10 +709,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title2 = "Device title 2"; String type2 = "typeD"; List devicesType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type2); @@ -775,4 +777,54 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(0, pageData.getData().size()); } + @Test + public void testAssignDeviceToTenant() throws Exception { + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + + Device anotherDevice = new Device(); + anotherDevice.setName("My device1"); + anotherDevice.setType("default"); + Device savedAnotherDevice = doPost("/api/device", anotherDevice, Device.class); + + EntityRelation relation = new EntityRelation(); + relation.setFrom(savedDevice.getId()); + relation.setTo(savedAnotherDevice.getId()); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relation.setType("Contains"); + doPost("/api/relation", relation).andExpect(status().isOk()); + + loginSysAdmin(); + Tenant tenant = new Tenant(); + tenant.setTitle("Different tenant"); + Tenant savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + Assert.assertNotNull(savedDifferentTenant); + + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setTenantId(savedDifferentTenant.getId()); + user.setEmail("tenant9@thingsboard.org"); + user.setFirstName("Sam"); + user.setLastName("Downs"); + + createUserAndLogin(user, "testPassword1"); + + login("tenant2@thingsboard.org", "testPassword1"); + Device assignedDevice = doPost("/api/tenant/" + savedDifferentTenant.getId().getId() + "/device/" + savedDevice.getId().getId(), Device.class); + + doGet("/api/device/" + assignedDevice.getId().getId().toString(), Device.class, status().isNotFound()); + + login("tenant9@thingsboard.org", "testPassword1"); + + Device foundDevice1 = doGet("/api/device/" + assignedDevice.getId().getId().toString(), Device.class); + Assert.assertNotNull(foundDevice1); + + doGet("/api/relation?fromId=" + savedDevice.getId().getId() + "&fromType=DEVICE&relationType=Contains&toId=" + savedAnotherDevice.getId().getId() + "&toType=DEVICE", EntityRelation.class, status().isNotFound()); + + loginSysAdmin(); + doDelete("/api/tenant/" + savedDifferentTenant.getId().getId().toString()) + .andExpect(status().isOk()); + } } diff --git a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java index ad81884f1f..a062fb7ebe 100644 --- a/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java +++ b/application/src/test/java/org/thingsboard/server/mqtt/MqttNoSqlTestSuite.java @@ -41,7 +41,9 @@ public class MqttNoSqlTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false)), + new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), + new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) + ), "cassandra-test.yaml", 30000l); @BeforeClass diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java index 572818d82b..c832d8ea06 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java @@ -16,14 +16,12 @@ package org.thingsboard.server.dao.audit; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; @@ -49,5 +47,4 @@ public interface AuditLogService { E entity, ActionType actionType, Exception e, Object... additionalInfo); - } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index c5e3ab7ca0..2480c67350 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -76,4 +76,6 @@ public interface DeviceService { ListenableFuture> findDeviceTypesByTenantId(TenantId tenantId); + Device assignDeviceToTenant(TenantId tenantId, Device device); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 425b3c4149..f1157e84d8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -41,4 +41,6 @@ public interface EventService { List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); + void removeEvents(TenantId tenantId, EntityId entityId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a54eecf55e..2e909b3460 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import java.util.List; -import java.util.concurrent.ExecutionException; /** * Created by ashvayka on 27.04.17. @@ -77,6 +76,8 @@ public interface RelationService { ListenableFuture> findInfoByQuery(TenantId tenantId, EntityRelationsQuery query); + void removeRelations(TenantId tenantId, EntityId entityId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java index bcf1222988..e267e5288d 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlAnyDao.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.entities.type}'=='cassandra'") +@ConditionalOnExpression("'${database.ts.type}'=='cassandra' || '${database.ts_latest.type}'=='cassandra'") public @interface NoSqlAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java new file mode 100644 index 0000000000..b9e83ddb56 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/NoSqlTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "cassandra") +public @interface NoSqlTsLatestDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java similarity index 86% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java index a215a16b29..9cf597e291 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsLatestAnyDao.java @@ -21,7 +21,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("('${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale') " + +@ConditionalOnExpression("('${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale') " + "&& '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") -public @interface PsqlTsAnyDao { +public @interface PsqlTsLatestAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java similarity index 83% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java index 95ca8c2b05..424e75ccdd 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/PsqlTsDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestAnyDao.java @@ -21,5 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='sql' && '${spring.jpa.database-platform}'=='org.hibernate.dialect.PostgreSQLDialect'") -public @interface PsqlTsDao { } \ No newline at end of file +@ConditionalOnExpression("'${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale'") +public @interface SqlTsLatestAnyDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java similarity index 79% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java index 76db7920a6..a2c07a43eb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsLatestDao.java @@ -15,9 +15,12 @@ */ package org.thingsboard.server.dao.util; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -public @interface SqlDao { +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "sql") +public @interface SqlTsLatestDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java similarity index 85% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java rename to common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java index 9be3321988..f2f5ecae4a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsAnyDao.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/SqlTsOrTsLatestAnyDao.java @@ -21,6 +21,6 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @Retention(RetentionPolicy.RUNTIME) -@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale'") -public @interface SqlTsAnyDao { +@ConditionalOnExpression("'${database.ts.type}'=='sql' || '${database.ts.type}'=='timescale' || '${database.ts_latest.type}'=='sql' || '${database.ts_latest.type}'=='timescale'") +public @interface SqlTsOrTsLatestAnyDao { } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java new file mode 100644 index 0000000000..579cd1a600 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnProperty(prefix = "database.ts_latest", value = "type", havingValue = "timescale") +public @interface TimescaleDBTsLatestDao { +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java new file mode 100644 index 0000000000..dcc44a0014 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/util/TimescaleDBTsOrTsLatestDao.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${database.ts.type}'=='timescale' || '${database.ts_latest.type}'=='timescale'") +public @interface TimescaleDBTsOrTsLatestDao { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index afb0dbeba6..e99db672a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -57,6 +57,8 @@ public class DataConstants { public static final String ATTRIBUTES_DELETED = "ATTRIBUTES_DELETED"; public static final String ALARM_ACK = "ALARM_ACK"; public static final String ALARM_CLEAR = "ALARM_CLEAR"; + public static final String ENTITY_ASSIGNED_FROM_TENANT = "ENTITY_ASSIGNED_FROM_TENANT"; + public static final String ENTITY_ASSIGNED_TO_TENANT = "ENTITY_ASSIGNED_TO_TENANT"; public static final String RPC_CALL_FROM_SERVER_TO_DEVICE = "RPC_CALL_FROM_SERVER_TO_DEVICE"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java index 03aeeba1d5..14a38c810c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java @@ -40,7 +40,9 @@ public enum ActionType { ALARM_CLEAR(false), LOGIN(false), LOGOUT(false), - LOCKOUT(false); + LOCKOUT(false), + ASSIGNED_FROM_TENANT(false), + ASSIGNED_TO_TENANT(false); private final boolean isRead; diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java index e2519191e9..3a54545952 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsDaoConfig.java @@ -27,8 +27,8 @@ import org.thingsboard.server.dao.util.SqlTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.hsql", "org.thingsboard.server.dao.sqlts.insert.latest.hsql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.hsql"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) @EnableTransactionManagement @SqlTsDao @HsqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java new file mode 100644 index 0000000000..c814baebfa --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/HsqlTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.HsqlDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.hsql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.hsql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@SqlTsLatestDao +@HsqlDao +public class HsqlTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java index 796f98d238..32db43271c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/JpaDaoConfig.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.transaction.annotation.EnableTransactionManagement; -import org.thingsboard.server.dao.util.SqlDao; /** * @author Valerii Sosliuk @@ -32,7 +31,6 @@ import org.thingsboard.server.dao.util.SqlDao; @EnableJpaRepositories("org.thingsboard.server.dao.sql") @EntityScan("org.thingsboard.server.dao.model.sql") @EnableTransactionManagement -@SqlDao public class JpaDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java index 65f17709ca..363b53713f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsDaoConfig.java @@ -27,11 +27,11 @@ import org.thingsboard.server.dao.util.SqlTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.psql"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.psql", "org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest", "org.thingsboard.server.dao.sqlts.dictionary"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts", "org.thingsboard.server.dao.model.sqlts.latest", "org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.ts", "org.thingsboard.server.dao.sqlts.insert.psql"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.ts"}) @EnableTransactionManagement -@SqlTsDao @PsqlDao +@SqlTsDao public class PsqlTsDaoConfig { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java new file mode 100644 index 0000000000..338bd71e73 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/PsqlTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.psql"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@SqlTsLatestDao +@PsqlDao +public class PsqlTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java new file mode 100644 index 0000000000..6b38df8284 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/SqlTimeseriesDaoConfig.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; + +@Configuration +@EnableAutoConfiguration +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.dictionary"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.dictionary"}) +@EnableTransactionManagement +@SqlTsOrTsLatestAnyDao +public class SqlTimeseriesDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java index 19ae98c736..673ff314c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleDaoConfig.java @@ -27,8 +27,8 @@ import org.thingsboard.server.dao.util.TimescaleDBTsDao; @Configuration @EnableAutoConfiguration @ComponentScan({"org.thingsboard.server.dao.sqlts.timescale"}) -@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.insert.timescale", "org.thingsboard.server.dao.sqlts.dictionary", "org.thingsboard.server.dao.sqlts.latest"}) -@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale", "org.thingsboard.server.dao.model.sqlts.dictionary", "org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.timescale", "org.thingsboard.server.dao.sqlts.insert.timescale"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.timescale"}) @EnableTransactionManagement @TimescaleDBTsDao @PsqlDao diff --git a/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java new file mode 100644 index 0000000000..9765dd6d11 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/TimescaleTsLatestDaoConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.EnableTransactionManagement; +import org.thingsboard.server.dao.util.PsqlDao; +import org.thingsboard.server.dao.util.TimescaleDBTsLatestDao; + +@Configuration +@EnableAutoConfiguration +@ComponentScan({"org.thingsboard.server.dao.sqlts.timescale"}) +@EnableJpaRepositories({"org.thingsboard.server.dao.sqlts.insert.latest.psql", "org.thingsboard.server.dao.sqlts.latest"}) +@EntityScan({"org.thingsboard.server.dao.model.sqlts.latest"}) +@EnableTransactionManagement +@TimescaleDBTsLatestDao +@PsqlDao +public class TimescaleTsLatestDaoConfig { + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 278c578e78..f0683ae5ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -163,7 +163,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(asset.getTenantId(), assetId).get(); if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete asset that is assigned to entity views!"); + throw new DataValidationException("Can't delete asset that has entity views!"); } } catch (ExecutionException | InterruptedException e) { log.error("Exception while finding entity views for assetId [{}]", assetId, e); diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java index 6b907093b3..34b5090179 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java @@ -23,11 +23,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.Dao; import java.util.List; import java.util.UUID; -public interface AuditLogDao { +public interface AuditLogDao extends Dao { ListenableFuture saveByTenantId(AuditLog auditLog); diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java index 5fe270dd31..467e957f74 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java @@ -163,6 +163,7 @@ public class AuditLogServiceImpl implements AuditLogService { case ALARM_ACK: case ALARM_CLEAR: case RELATIONS_DELETED: + case ASSIGNED_TO_TENANT: if (entity != null) { ObjectNode entityNode = objectMapper.valueToTree(entity); if (entityId.getEntityType() == EntityType.DASHBOARD) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java index a3863ed012..12acd08361 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java @@ -58,5 +58,4 @@ public class DummyAuditLogServiceImpl implements AuditLogService { public ListenableFuture> logEntityAction(TenantId tenantId, CustomerId customerId, UserId userId, String userName, I entityId, E entity, ActionType actionType, Exception e, Object... additionalInfo) { return null; } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 56666d9c0d..ae87564e81 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -166,4 +166,21 @@ public interface DeviceDao extends Dao { * @return the list of tenant device type objects */ ListenableFuture> findTenantDeviceTypesAsync(UUID tenantId); + + /** + * Find devices by tenantId and device id. + * @param tenantId the tenant Id + * @param id the device Id + * @return the device object + */ + Device findDeviceByTenantIdAndId(TenantId tenantId, UUID id); + + /** + * Find devices by tenantId and device id. + * @param tenantId tenantId the tenantId + * @param id the deviceId + * @return the device object + */ + ListenableFuture findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 4a39939f1e..219656badd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -28,6 +28,8 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -50,6 +52,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -98,6 +101,9 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Autowired private CacheManager cacheManager; + @Autowired + private EventService eventService; + @Override public DeviceInfo findDeviceInfoById(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceInfoById [{}]", deviceId); @@ -109,14 +115,22 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe public Device findDeviceById(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); - return deviceDao.findById(tenantId, deviceId.getId()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return deviceDao.findById(tenantId, deviceId.getId()); + } else { + return deviceDao.findDeviceByTenantIdAndId(tenantId, deviceId.getId()); + } } @Override public ListenableFuture findDeviceByIdAsync(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); - return deviceDao.findByIdAsync(tenantId, deviceId.getId()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return deviceDao.findByIdAsync(tenantId, deviceId.getId()); + } else { + return deviceDao.findDeviceByTenantIdAndIdAsync(tenantId, deviceId.getId()); + } } @Cacheable(cacheNames = DEVICE_CACHE, key = "{#tenantId, #name}") @@ -187,7 +201,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete device that is assigned to entity views!"); + throw new DataValidationException("Can't delete device that has entity views!"); } } catch (ExecutionException | InterruptedException e) { log.error("Exception while finding entity views for deviceId [{}]", deviceId, e); @@ -353,6 +367,31 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe }, MoreExecutors.directExecutor()); } + @Transactional + @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}") + @Override + public Device assignDeviceToTenant(TenantId tenantId, Device device) { + log.trace("Executing assignDeviceToTenant [{}][{}]", tenantId, device); + + try { + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); + if (!CollectionUtils.isEmpty(entityViews)) { + throw new DataValidationException("Can't assign device that has entity views to another tenant!"); + } + } catch (ExecutionException | InterruptedException e) { + log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); + throw new RuntimeException("Exception while finding entity views for deviceId [" + device.getId() + "]", e); + } + + eventService.removeEvents(device.getTenantId(), device.getId()); + + relationService.removeRelations(device.getTenantId(), device.getId()); + + device.setTenantId(tenantId); + device.setCustomerId(null); + return doSaveDevice(device, null); + } + private DataValidator deviceValidator = new DataValidator() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index f1ccde4415..ef5fcf0fa2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -92,6 +92,21 @@ public class BaseEventService implements EventService { return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit); } + @Override + public void removeEvents(TenantId tenantId, EntityId entityId) { + PageData eventPageData; + TimePageLink eventPageLink = new TimePageLink(1000); + do { + eventPageData = findEvents(tenantId, entityId, eventPageLink); + for (Event event : eventPageData.getData()) { + eventDao.removeById(tenantId, event.getUuidId()); + } + if (eventPageData.hasNext()) { + eventPageLink = eventPageLink.nextPageLink(); + } + } while (eventPageData.hasNext()); + } + private DataValidator eventValidator = new DataValidator() { @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 22b2543489..53ca684517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -506,6 +506,22 @@ public class BaseRelationService implements RelationService { }, MoreExecutors.directExecutor()); } + @Override + public void removeRelations(TenantId tenantId, EntityId entityId) { + Cache cache = cacheManager.getCache(RELATIONS_CACHE); + + List relations = new ArrayList<>(); + for (RelationTypeGroup relationTypeGroup : RelationTypeGroup.values()) { + relations.addAll(findByFrom(tenantId, entityId, relationTypeGroup)); + relations.addAll(findByTo(tenantId, entityId, relationTypeGroup)); + } + + for (EntityRelation relation : relations) { + cacheEviction(relation, cache); + deleteRelation(tenantId, relation); + } + } + protected void validate(EntityRelation relation) { if (relation == null) { throw new DataValidationException("Relation type should be specified!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java index ef9d287299..ace621cd6a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaExecutorService.java @@ -18,10 +18,8 @@ package org.thingsboard.server.dao.sql; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.thingsboard.common.util.AbstractListeningExecutor; -import org.thingsboard.server.dao.util.SqlDao; @Component -@SqlDao public class JpaExecutorService extends AbstractListeningExecutor { @Value("${spring.datasource.hikari.maximumPoolSize}") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 3c85cf2c60..1f82329c90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -21,25 +21,15 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.alarm.AlarmStatus; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.query.AlarmData; -import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.model.sql.AlarmInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; -import java.util.Collection; import java.util.List; -import java.util.Set; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -@SqlDao public interface AlarmRepository extends CrudRepository { @Query("SELECT a FROM AlarmEntity a WHERE a.originatorId = :originatorId AND a.type = :alarmType ORDER BY a.startTs DESC") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 2b42f81e6e..202281288b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -39,7 +39,6 @@ import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.query.AlarmQueryRepository; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collection; @@ -54,7 +53,6 @@ import java.util.UUID; */ @Slf4j @Component -@SqlDao public class JpaAlarmDao extends JpaAbstractDao implements AlarmDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 6d87d75366..5e9d942720 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -@SqlDao public interface AssetRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index e836aa1ef2..1e67802873 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.asset.AssetDao; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -44,7 +43,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/19/2017. */ @Component -@SqlDao public class JpaAssetDao extends JpaAbstractSearchTextDao implements AssetDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java index 70500d38a0..01bfaa99b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvInsertRepository.java @@ -25,7 +25,6 @@ import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -35,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.regex.Pattern; -@SqlDao @Repository @Slf4j public abstract class AttributeKvInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java index 1114fe5e66..f3ef44e6c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/AttributeKvRepository.java @@ -23,12 +23,10 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; -@SqlDao public interface AttributeKvRepository extends CrudRepository { @Query("SELECT a FROM AttributeKvEntity a WHERE a.id.entityType = :entityType " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java index ce90a2d8cd..718b9e49dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/HsqlAttributesInsertRepository.java @@ -19,12 +19,10 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import java.sql.Types; import java.util.List; -@SqlDao @HsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index dedec0bc07..2803dec8dd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -34,7 +34,6 @@ import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; -import org.thingsboard.server.dao.util.SqlDao; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -46,7 +45,6 @@ import java.util.stream.Collectors; @Component @Slf4j -@SqlDao public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService implements AttributesDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java index 020e63cd36..076a472da1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/PsqlAttributesInsertRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sql.attributes; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao @PsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java index 1ffce82735..714b6eebd4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/audit/JpaAuditLogDao.java @@ -30,14 +30,12 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.audit.AuditLogDao; import org.thingsboard.server.dao.model.sql.AuditLogEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.Objects; import java.util.UUID; @Component -@SqlDao public class JpaAuditLogDao extends JpaAbstractDao implements AuditLogDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java index 83c2ad1e23..668b31c84a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java @@ -23,14 +23,12 @@ import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface ComponentDescriptorRepository extends PagingAndSortingRepository { ComponentDescriptorEntity findByClazz(String clazz); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java index 7477da2f74..05e28f637c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/HsqlComponentDescriptorInsertRepository.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; -@SqlDao @HsqlDao @Repository public class HsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java index c128eda12f..d27eb7d3d1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/JpaBaseComponentDescriptorDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.component.ComponentDescriptorDao; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.Optional; @@ -41,7 +40,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaBaseComponentDescriptorDao extends JpaAbstractSearchTextDao implements ComponentDescriptorDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java index dfe63ec69a..44a1cf8312 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/PsqlComponentDescriptorInsertRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao @PsqlDao @Repository public class PsqlComponentDescriptorInsertRepository extends AbstractComponentDescriptorInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 66977419fe..bf2a845ec0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.CustomerEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface CustomerRepository extends PagingAndSortingRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index a3b585c596..14c025de42 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.model.sql.CustomerEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.Optional; @@ -35,7 +34,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaCustomerDao extends JpaAbstractSearchTextDao implements CustomerDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java index f2e21f06c5..b0eb8f4d7b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardInfoRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.DashboardInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DashboardInfoRepository extends PagingAndSortingRepository { @Query("SELECT di FROM DashboardInfoEntity di WHERE di.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 76b8723f2a..bc2dcf19f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -17,13 +17,11 @@ package org.thingsboard.server.dao.sql.dashboard; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DashboardRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 017cf2e328..8d637f0cbe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDashboardDao extends JpaAbstractSearchTextDao implements DashboardDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java index ac8fb2aabc..6fadc233d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardInfoDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.dashboard.DashboardInfoDao; import org.thingsboard.server.dao.model.sql.DashboardInfoEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -37,7 +36,6 @@ import java.util.UUID; */ @Slf4j @Component -@SqlDao public class JpaDashboardInfoDao extends JpaAbstractSearchTextDao implements DashboardInfoDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 8995a33b3a..c4577dd1e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -17,14 +17,12 @@ package org.thingsboard.server.dao.sql.device; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DeviceCredentialsRepository extends CrudRepository { DeviceCredentialsEntity findByDeviceId(UUID deviceId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index d198333ef0..828a9e28f0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -@SqlDao public interface DeviceRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo) " + @@ -127,4 +125,7 @@ public interface DeviceRepository extends PagingAndSortingRepository findDevicesByTenantIdAndCustomerIdAndIdIn(UUID tenantId, UUID customerId, List deviceIds); List findDevicesByTenantIdAndIdIn(UUID tenantId, List deviceIds); + + DeviceEntity findByTenantIdAndId(UUID tenantId, UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 12a38d2642..a2cde30fd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.device.DeviceCredentialsDao; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDeviceCredentialsDao extends JpaAbstractDao implements DeviceCredentialsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 4942df5fa1..e8e36a7fd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -32,7 +32,6 @@ import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -45,7 +44,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 5/6/2017. */ @Component -@SqlDao public class JpaDeviceDao extends JpaAbstractSearchTextDao implements DeviceDao { @Autowired @@ -175,6 +173,16 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return service.submit(() -> convertTenantDeviceTypesToDto(tenantId, deviceRepository.findTenantDeviceTypes(tenantId))); } + @Override + public Device findDeviceByTenantIdAndId(TenantId tenantId, UUID id) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndId(tenantId.getId(), id)); + } + + @Override + public ListenableFuture findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id) { + return service.submit(() -> DaoUtil.getData(deviceRepository.findByTenantIdAndId(tenantId.getId(), id))); + } + private List convertTenantDeviceTypesToDto(UUID tenantId, List types) { List list = Collections.emptyList(); if (types != null && !types.isEmpty()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index d119648d76..0c7c55426e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/31/2017. */ -@SqlDao public interface EntityViewRepository extends PagingAndSortingRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.EntityViewInfoEntity(e, c.title, c.additionalInfo) " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 556d8dab5e..0e42b579fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Collections; @@ -44,7 +43,6 @@ import java.util.UUID; * Created by Victor Basanets on 8/31/2017. */ @Component -@SqlDao public class JpaEntityViewDao extends JpaAbstractSearchTextDao implements EntityViewDao { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java index 6a09a70ebc..5696bfbf50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/EventRepository.java @@ -22,7 +22,6 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.dao.model.sql.EventEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -30,7 +29,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/3/2017. */ -@SqlDao public interface EventRepository extends PagingAndSortingRepository { EventEntity findByTenantIdAndEntityTypeAndEntityIdAndEventTypeAndEventUid(UUID tenantId, diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java index 1122499721..722c370ac9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/HsqlEventInsertRepository.java @@ -18,11 +18,9 @@ package org.thingsboard.server.dao.sql.event; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; -@SqlDao @HsqlDao @Repository public class HsqlEventInsertRepository extends AbstractEventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java index c9e9750a5f..20f49ac8d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/JpaBaseEventDao.java @@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; -import org.springframework.data.jpa.domain.Specification; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Event; @@ -34,10 +33,7 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.event.EventDao; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; -import javax.persistence.criteria.Predicate; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -50,7 +46,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; */ @Slf4j @Component -@SqlDao public class JpaBaseEventDao extends JpaAbstractDao implements EventDao { private final UUID systemTenantId = NULL_UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java index e0e3e728ea..1f401e5eb2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/PsqlEventInsertRepository.java @@ -19,10 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sql.EventEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @Slf4j -@SqlDao @PsqlDao @Repository public class PsqlEventInsertRepository extends AbstractEventInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index f33667271b..29b070c9e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.util.SqlDao; import java.util.ArrayList; import java.util.Arrays; @@ -52,7 +51,6 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -@SqlDao @Repository @Slf4j public class DefaultAlarmQueryRepository implements AlarmQueryRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 06a6f7f059..7e088f2016 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -50,7 +50,6 @@ import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.EntityTypeFilter; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; import java.util.Collections; @@ -61,7 +60,6 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; -@SqlDao @Repository @Slf4j public class DefaultEntityQueryRepository implements EntityQueryRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java index 7e8e77e25f..2c77c5d833 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/JpaEntityQueryDao.java @@ -24,10 +24,8 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.dao.entity.EntityQueryDao; -import org.thingsboard.server.dao.util.SqlDao; @Component -@SqlDao public class JpaEntityQueryDao implements EntityQueryDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java index fc262945e1..e017831607 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/HsqlRelationInsertRepository.java @@ -20,12 +20,10 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.Query; @HsqlDao -@SqlDao @Repository @Transactional public class HsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index b96d3db110..3b6e7a83cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -30,7 +30,6 @@ import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; -import org.thingsboard.server.dao.util.SqlDao; import javax.persistence.criteria.Predicate; import java.util.ArrayList; @@ -41,7 +40,6 @@ import java.util.List; */ @Slf4j @Component -@SqlDao public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java index dbef233811..7325fc0fbc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/PsqlRelationInsertRepository.java @@ -19,10 +19,8 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.util.PsqlDao; -import org.thingsboard.server.dao.util.SqlDao; @PsqlDao -@SqlDao @Repository @Transactional public class PsqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index 95c67e0b9f..ada7a44565 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -20,12 +20,10 @@ import org.springframework.data.repository.CrudRepository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RelationCompositeKey; import org.thingsboard.server.dao.model.sql.RelationEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; -@SqlDao public interface RelationRepository extends CrudRepository, JpaSpecificationExecutor { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 63e2889bc5..57d1545a94 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -26,14 +26,12 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.RuleChainEntity; import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @Slf4j @Component -@SqlDao public class JpaRuleChainDao extends JpaAbstractSearchTextDao implements RuleChainDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index e29a988fc1..e5302e2618 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -23,11 +23,9 @@ import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import org.thingsboard.server.dao.rule.RuleNodeDao; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; @Slf4j @Component -@SqlDao public class JpaRuleNodeDao extends JpaAbstractSearchTextDao implements RuleNodeDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 24b795cae4..228d4d26f4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -21,11 +21,9 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.RuleChainEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; -@SqlDao public interface RuleChainRepository extends PagingAndSortingRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index d3f51ec4f6..58982f6f73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -17,9 +17,7 @@ package org.thingsboard.server.dao.sql.rule; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao public interface RuleNodeRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index ad56b5754c..4e1d44d7f2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -25,13 +25,11 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AdminSettingsEntity; import org.thingsboard.server.dao.settings.AdminSettingsDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @Component @Slf4j -@SqlDao public class JpaAdminSettingsDao extends JpaAbstractDao implements AdminSettingsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index 2c1dcb7516..6f22af118c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -26,7 +26,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.TenantEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -36,7 +35,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/30/2017. */ @Component -@SqlDao public class JpaTenantDao extends JpaAbstractSearchTextDao implements TenantDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java index 04471d2e26..012920e187 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/TenantRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.TenantEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/30/2017. */ -@SqlDao public interface TenantRepository extends PagingAndSortingRepository { @Query("SELECT t FROM TenantEntity t WHERE t.region = :region " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java index 5c732bafd8..87e5a496d9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserCredentialsDao.java @@ -24,7 +24,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.user.UserCredentialsDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/22/2017. */ @Component -@SqlDao public class JpaUserCredentialsDao extends JpaAbstractDao implements UserCredentialsDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index b0805c1a52..42092f7ebe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -27,7 +27,6 @@ import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.UserEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import org.thingsboard.server.dao.user.UserDao; -import org.thingsboard.server.dao.util.SqlDao; import java.util.Objects; import java.util.UUID; @@ -38,7 +37,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; * @author Valerii Sosliuk */ @Component -@SqlDao public class JpaUserDao extends JpaAbstractSearchTextDao implements UserDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java index 57e954f796..4b1791ddcb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserCredentialsRepository.java @@ -17,14 +17,12 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/22/2017. */ -@SqlDao public interface UserCredentialsRepository extends CrudRepository { UserCredentialsEntity findByUserId(UUID userId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 5bae8fbe68..1d512a7944 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -22,14 +22,12 @@ import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.model.sql.UserEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * @author Valerii Sosliuk */ -@SqlDao public interface UserRepository extends PagingAndSortingRepository { UserEntity findByEmail(String email); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index 4be5e5f514..805897caf4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetTypeDao; import java.util.List; @@ -32,7 +31,6 @@ import java.util.UUID; * Created by Valerii Sosliuk on 4/29/2017. */ @Component -@SqlDao public class JpaWidgetTypeDao extends JpaAbstractDao implements WidgetTypeDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java index 8e84f95627..e0611a6263 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetsBundleDao.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; -import org.thingsboard.server.dao.util.SqlDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; import java.util.Objects; @@ -37,7 +36,6 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; * Created by Valerii Sosliuk on 4/23/2017. */ @Component -@SqlDao public class JpaWidgetsBundleDao extends JpaAbstractSearchTextDao implements WidgetsBundleDao { @Autowired diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java index 387f21d7d7..6167e64234 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sql.WidgetTypeEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; @@ -25,7 +24,6 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 4/29/2017. */ -@SqlDao public interface WidgetTypeRepository extends CrudRepository { List findByTenantIdAndBundleAlias(UUID tenantId, String bundleAlias); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index f442a9739e..b4030f90de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -21,14 +21,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.UUID; /** * Created by Valerii Sosliuk on 4/23/2017. */ -@SqlDao public interface WidgetsBundleRepository extends PagingAndSortingRepository { WidgetsBundleEntity findWidgetsBundleByTenantIdAndAlias(UUID tenantId, String alias); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 1d8194d805..079bf1b660 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -17,7 +17,6 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; @@ -33,7 +32,6 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueue; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.insert.InsertTsRepository; @@ -64,7 +62,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @PostConstruct protected void init() { - super.init(); TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() .logName("TS") .batchSize(tsBatchSize) @@ -80,7 +77,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq @PreDestroy protected void destroy() { - super.destroy(); if (tsQueue != null) { tsQueue.destroy(); } @@ -98,26 +94,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq }); } - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return getSaveLatestFuture(entityId, tsKvEntry); - } - - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return getRemoveLatestFuture(entityId, query); - } - - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return getFindLatestFuture(entityId, key); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return getFindAllLatestFuture(entityId); - } - @Override public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { return Futures.immediateFuture(null); @@ -134,7 +110,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } @Override - protected ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(entityId, query); } else { @@ -151,8 +127,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } } - @Override - protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 9710edad4a..9ab8aa3dfc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -16,96 +16,28 @@ package org.thingsboard.server.dao.sqlts; import com.google.common.base.Function; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.Aggregation; -import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; -import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; -import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; -import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; -import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; -import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; -import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; import javax.annotation.Nullable; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { - - private static final String DESC_ORDER = "DESC"; - - private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - - private static final ReentrantLock tsCreationLock = new ReentrantLock(); - - @Autowired - private TsKvLatestRepository tsKvLatestRepository; - - @Autowired - private SearchTsKvLatestRepository searchTsKvLatestRepository; - - @Autowired - private InsertLatestTsRepository insertLatestTsRepository; - - @Autowired - private TsKvDictionaryRepository dictionaryRepository; - - private TbSqlBlockingQueueWrapper tsLatestQueue; - - @Value("${sql.ts_latest.batch_size:1000}") - private int tsLatestBatchSize; - - @Value("${sql.ts_latest.batch_max_delay:100}") - private long tsLatestMaxDelay; - - @Value("${sql.ts_latest.stats_print_interval_ms:1000}") - private long tsLatestStatsPrintIntervalMs; - - @Value("${sql.ts_latest.batch_threads:4}") - private int tsLatestBatchThreads; +public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { @Autowired protected ScheduledLogExecutorComponent logExecutor; - @Autowired - private StatsFactory statsFactory; - @Value("${sql.ts.batch_size:1000}") protected int tsBatchSize; @@ -121,44 +53,10 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx @Value("${sql.timescale.batch_threads:4}") protected int timescaleBatchThreads; - @PostConstruct - protected void init() { - TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() - .logName("TS Latest") - .batchSize(tsLatestBatchSize) - .maxDelay(tsLatestMaxDelay) - .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) - .statsNamePrefix("ts.latest") - .build(); - - java.util.function.Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); - tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory); - - tsLatestQueue.init(logExecutor, v -> { - Map trueLatest = new HashMap<>(); - v.forEach(ts -> { - TsKey key = new TsKey(ts.getEntityId(), ts.getKey()); - TsKvLatestEntity old = trueLatest.get(key); - if (old == null || old.getTs() < ts.getTs()) { - trueLatest.put(key, ts); - } - }); - List latestEntities = new ArrayList<>(trueLatest.values()); - insertLatestTsRepository.saveOrUpdate(latestEntities); - }); - } - - @PreDestroy - protected void destroy() { - if (tsLatestQueue != null) { - tsLatestQueue.destroy(); - } - } - protected ListenableFuture> processFindAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries .stream() - .map(query -> findAllAsync(entityId, query)) + .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); return Futures.transform(Futures.allAsList(futures), new Function>, List>() { @Nullable @@ -174,168 +72,4 @@ public abstract class AbstractSqlTimeseriesDao extends JpaAbstractDaoListeningEx } }, service); } - - protected abstract ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query); - - protected abstract ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query); - - protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { - return Futures.transform(future, new Function>, List>() { - @Nullable - @Override - public List apply(@Nullable List> results) { - if (results == null || results.isEmpty()) { - return null; - } - return results.stream() - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.toList()); - } - }, service); - } - - protected ListenableFuture> findNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) { - long startTs = 0; - long endTs = query.getStartTs() - 1; - ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, - Aggregation.NONE, DESC_ORDER); - return findAllAsync(entityId, findNewLatestQuery); - } - - protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { - TsKvLatestCompositeKey compositeKey = - new TsKvLatestCompositeKey( - entityId.getId(), - getOrSaveKeyId(key)); - Optional entry = tsKvLatestRepository.findById(compositeKey); - TsKvEntry result; - if (entry.isPresent()) { - TsKvLatestEntity tsKvLatestEntity = entry.get(); - tsKvLatestEntity.setStrKey(key); - result = DaoUtil.getData(tsKvLatestEntity); - } else { - result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - return Futures.immediateFuture(result); - } - - protected ListenableFuture getRemoveLatestFuture(EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestFuture = getFindLatestFuture(entityId, query.getKey()); - - ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { - long ts = tsKvEntry.getTs(); - return ts > query.getStartTs() && ts <= query.getEndTs(); - }, service); - - ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityId(entityId.getId()); - latestEntity.setKey(getOrSaveKeyId(query.getKey())); - return service.submit(() -> { - tsKvLatestRepository.delete(latestEntity); - return null; - }); - } - return Futures.immediateFuture(null); - }, service); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(removedLatestFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - if (query.getRewriteLatestIfDeleted()) { - ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return getNewLatestEntryFuture(entityId, query); - } - return Futures.immediateFuture(null); - }, service); - - try { - resultFuture.set(savedLatestFuture.get()); - } catch (InterruptedException | ExecutionException e) { - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); - } - } else { - resultFuture.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to process remove of the latest value", entityId, t); - } - }, MoreExecutors.directExecutor()); - return resultFuture; - } - - protected ListenableFuture> getFindAllLatestFuture(EntityId entityId) { - return Futures.immediateFuture( - DaoUtil.convertDataList(Lists.newArrayList( - searchTsKvLatestRepository.findAllByEntityId(entityId.getId())))); - } - - protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { - TsKvLatestEntity latestEntity = new TsKvLatestEntity(); - latestEntity.setEntityId(entityId.getId()); - latestEntity.setTs(tsKvEntry.getTs()); - latestEntity.setKey(getOrSaveKeyId(tsKvEntry.getKey())); - latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - latestEntity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - - return tsLatestQueue.add(latestEntity); - } - - protected Integer getOrSaveKeyId(String strKey) { - Integer keyId = tsKvDictionaryMap.get(strKey); - if (keyId == null) { - Optional tsKvDictionaryOptional; - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - if (!tsKvDictionaryOptional.isPresent()) { - tsCreationLock.lock(); - try { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - if (!tsKvDictionaryOptional.isPresent()) { - TsKvDictionary tsKvDictionary = new TsKvDictionary(); - tsKvDictionary.setKey(strKey); - try { - TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); - tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); - keyId = saved.getKeyId(); - } catch (ConstraintViolationException e) { - tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); - TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); - tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); - keyId = dictionary.getKeyId(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - } - } finally { - tsCreationLock.unlock(); - } - } else { - keyId = tsKvDictionaryOptional.get().getKeyId(); - tsKvDictionaryMap.put(strKey, keyId); - } - } - return keyId; - } - - private ListenableFuture getNewLatestEntryFuture(EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture> future = findNewLatestEntryFuture(entityId, query); - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return getSaveLatestFuture(entityId, entryList.get(0)); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, service); - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java new file mode 100644 index 0000000000..345dc26e48 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface AggregationTimeseriesDao { + + ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java new file mode 100644 index 0000000000..1d7fbbaac0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.base.Function; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.hibernate.exception.ConstraintViolationException; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; +import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; +import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import org.thingsboard.server.dao.sqlts.dictionary.TsKvDictionaryRepository; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Slf4j +public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { + + private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); + + protected static final ReentrantLock tsCreationLock = new ReentrantLock(); + + @Autowired + protected TsKvDictionaryRepository dictionaryRepository; + + protected Integer getOrSaveKeyId(String strKey) { + Integer keyId = tsKvDictionaryMap.get(strKey); + if (keyId == null) { + Optional tsKvDictionaryOptional; + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + tsCreationLock.lock(); + try { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + if (!tsKvDictionaryOptional.isPresent()) { + TsKvDictionary tsKvDictionary = new TsKvDictionary(); + tsKvDictionary.setKey(strKey); + try { + TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); + tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); + keyId = saved.getKeyId(); + } catch (ConstraintViolationException e) { + tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); + TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); + tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); + keyId = dictionary.getKeyId(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + } + } finally { + tsCreationLock.unlock(); + } + } else { + keyId = tsKvDictionaryOptional.get().getKeyId(); + tsKvDictionaryMap.put(strKey, keyId); + } + } + return keyId; + } + + protected ListenableFuture> getTskvEntriesFuture(ListenableFuture>> future) { + return Futures.transform(future, new Function>, List>() { + @Nullable + @Override + public List apply(@Nullable List> results) { + if (results == null || results.isEmpty()) { + return null; + } + return results.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + }, service); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java new file mode 100644 index 0000000000..ee7c02d558 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -0,0 +1,261 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; +import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; +import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; +import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; +import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; +import org.thingsboard.server.dao.sqlts.latest.SearchTsKvLatestRepository; +import org.thingsboard.server.dao.sqlts.latest.TsKvLatestRepository; +import org.thingsboard.server.dao.timeseries.SimpleListenableFuture; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; + +import javax.annotation.Nullable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +@Slf4j +@Component +@SqlTsLatestAnyDao +public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesLatestDao { + + private static final String DESC_ORDER = "DESC"; + + @Autowired + private TsKvLatestRepository tsKvLatestRepository; + + @Autowired + protected AggregationTimeseriesDao aggregationTimeseriesDao; + + @Autowired + private SearchTsKvLatestRepository searchTsKvLatestRepository; + + @Autowired + private InsertLatestTsRepository insertLatestTsRepository; + + private TbSqlBlockingQueueWrapper tsLatestQueue; + + @Value("${sql.ts_latest.batch_size:1000}") + private int tsLatestBatchSize; + + @Value("${sql.ts_latest.batch_max_delay:100}") + private long tsLatestMaxDelay; + + @Value("${sql.ts_latest.stats_print_interval_ms:1000}") + private long tsLatestStatsPrintIntervalMs; + + @Value("${sql.ts_latest.batch_threads:4}") + private int tsLatestBatchThreads; + + @Autowired + protected ScheduledLogExecutorComponent logExecutor; + + @Autowired + private StatsFactory statsFactory; + + @PostConstruct + protected void init() { + TbSqlBlockingQueueParams tsLatestParams = TbSqlBlockingQueueParams.builder() + .logName("TS Latest") + .batchSize(tsLatestBatchSize) + .maxDelay(tsLatestMaxDelay) + .statsPrintIntervalMs(tsLatestStatsPrintIntervalMs) + .statsNamePrefix("ts.latest") + .build(); + + java.util.function.Function hashcodeFunction = entity -> entity.getEntityId().hashCode(); + tsLatestQueue = new TbSqlBlockingQueueWrapper<>(tsLatestParams, hashcodeFunction, tsLatestBatchThreads, statsFactory); + + tsLatestQueue.init(logExecutor, v -> { + Map trueLatest = new HashMap<>(); + v.forEach(ts -> { + TsKey key = new TsKey(ts.getEntityId(), ts.getKey()); + TsKvLatestEntity old = trueLatest.get(key); + if (old == null || old.getTs() < ts.getTs()) { + trueLatest.put(key, ts); + } + }); + List latestEntities = new ArrayList<>(trueLatest.values()); + insertLatestTsRepository.saveOrUpdate(latestEntities); + }); + } + + @PreDestroy + protected void destroy() { + if (tsLatestQueue != null) { + tsLatestQueue.destroy(); + } + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + return getSaveLatestFuture(entityId, tsKvEntry); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return getRemoveLatestFuture(tenantId, entityId, query); + } + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + return getFindLatestFuture(entityId, key); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + return getFindAllLatestFuture(entityId); + } + + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); + return Futures.transformAsync(future, entryList -> { + if (entryList.size() == 1) { + return getSaveLatestFuture(entityId, entryList.get(0)); + } else { + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, service); + } + + private ListenableFuture> findNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long startTs = 0; + long endTs = query.getStartTs() - 1; + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, + Aggregation.NONE, DESC_ORDER); + return aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + } + + protected ListenableFuture getFindLatestFuture(EntityId entityId, String key) { + TsKvLatestCompositeKey compositeKey = + new TsKvLatestCompositeKey( + entityId.getId(), + getOrSaveKeyId(key)); + Optional entry = tsKvLatestRepository.findById(compositeKey); + TsKvEntry result; + if (entry.isPresent()) { + TsKvLatestEntity tsKvLatestEntity = entry.get(); + tsKvLatestEntity.setStrKey(key); + result = DaoUtil.getData(tsKvLatestEntity); + } else { + result = new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + } + return Futures.immediateFuture(result); + } + + protected ListenableFuture getRemoveLatestFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture latestFuture = getFindLatestFuture(entityId, query.getKey()); + + ListenableFuture booleanFuture = Futures.transform(latestFuture, tsKvEntry -> { + long ts = tsKvEntry.getTs(); + return ts > query.getStartTs() && ts <= query.getEndTs(); + }, service); + + ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(entityId.getId()); + latestEntity.setKey(getOrSaveKeyId(query.getKey())); + return service.submit(() -> { + tsKvLatestRepository.delete(latestEntity); + return null; + }); + } + return Futures.immediateFuture(null); + }, service); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + Futures.addCallback(removedLatestFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + if (query.getRewriteLatestIfDeleted()) { + ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return getNewLatestEntryFuture(tenantId, entityId, query); + } + return Futures.immediateFuture(null); + }, service); + + try { + resultFuture.set(savedLatestFuture.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); + } + } else { + resultFuture.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to process remove of the latest value", entityId, t); + } + }, MoreExecutors.directExecutor()); + return resultFuture; + } + + protected ListenableFuture> getFindAllLatestFuture(EntityId entityId) { + return Futures.immediateFuture( + DaoUtil.convertDataList(Lists.newArrayList( + searchTsKvLatestRepository.findAllByEntityId(entityId.getId())))); + } + + protected ListenableFuture getSaveLatestFuture(EntityId entityId, TsKvEntry tsKvEntry) { + TsKvLatestEntity latestEntity = new TsKvLatestEntity(); + latestEntity.setEntityId(entityId.getId()); + latestEntity.setTs(tsKvEntry.getTs()); + latestEntity.setKey(getOrSaveKeyId(tsKvEntry.getKey())); + latestEntity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + latestEntity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + latestEntity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + latestEntity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + latestEntity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); + + return tsLatestQueue.add(latestEntity); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java index 55d2d031d9..76872ab176 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/dictionary/TsKvDictionaryRepository.java @@ -18,11 +18,11 @@ package org.thingsboard.server.dao.sqlts.dictionary; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; -import org.thingsboard.server.dao.util.SqlTsAnyDao; +import org.thingsboard.server.dao.util.SqlTsOrTsLatestAnyDao; import java.util.Optional; -@SqlTsAnyDao +@SqlTsOrTsLatestAnyDao public interface TsKvDictionaryRepository extends CrudRepository { Optional findByKeyId(int keyId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java index 224dc52805..2b6c2b9012 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java @@ -22,14 +22,14 @@ import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; import org.thingsboard.server.dao.util.HsqlDao; -import org.thingsboard.server.dao.util.SqlTsDao; +import org.thingsboard.server.dao.util.SqlTsLatestDao; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.util.List; -@SqlTsDao +@SqlTsLatestDao @HsqlDao @Repository @Transactional diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java index 41abae52f8..034ef26c00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/psql/PsqlLatestInsertTsRepository.java @@ -23,7 +23,7 @@ import org.springframework.transaction.support.TransactionCallbackWithoutResult; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.sqlts.insert.AbstractInsertRepository; import org.thingsboard.server.dao.sqlts.insert.latest.InsertLatestTsRepository; -import org.thingsboard.server.dao.util.PsqlTsAnyDao; +import org.thingsboard.server.dao.util.PsqlTsLatestAnyDao; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -32,7 +32,7 @@ import java.util.ArrayList; import java.util.List; -@PsqlTsAnyDao +@PsqlTsLatestAnyDao @Repository @Transactional public class PsqlLatestInsertTsRepository extends AbstractInsertRepository implements InsertLatestTsRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java index b5da093b6f..19687a94ca 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/SearchTsKvLatestRepository.java @@ -17,14 +17,14 @@ package org.thingsboard.server.dao.sqlts.latest; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.util.SqlTsAnyDao; +import org.thingsboard.server.dao.util.SqlTsLatestAnyDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import java.util.List; import java.util.UUID; -@SqlTsAnyDao +@SqlTsLatestAnyDao @Repository public class SearchTsKvLatestRepository { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java index 5232b88489..bb53106254 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/latest/TsKvLatestRepository.java @@ -18,9 +18,7 @@ package org.thingsboard.server.dao.sqlts.latest; import org.springframework.data.repository.CrudRepository; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestCompositeKey; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; -import org.thingsboard.server.dao.util.SqlDao; -@SqlDao public interface TsKvLatestRepository extends CrudRepository { } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java index 7d0aefb62c..8c50bcb38c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/psql/JpaPsqlTimeseriesDao.java @@ -41,11 +41,10 @@ import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; - @Component @Slf4j -@SqlTsDao @PsqlDao +@SqlTsDao public class JpaPsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao { private final Map partitions = new ConcurrentHashMap<>(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java index 28b666b03b..e12a0219d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/AggregationRepository.java @@ -18,7 +18,7 @@ package org.thingsboard.server.dao.sqlts.timescale; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Repository; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.util.TimescaleDBTsDao; +import org.thingsboard.server.dao.util.TimescaleDBTsOrTsLatestDao; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -27,7 +27,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; @Repository -@TimescaleDBTsDao +@TimescaleDBTsOrTsLatestDao public class AggregationRepository { public static final String FIND_AVG = "findAvg"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 738c98d3b7..71647f88f9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -72,7 +72,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @PostConstruct protected void init() { - super.init(); TbSqlBlockingQueueParams tsParams = TbSqlBlockingQueueParams.builder() .logName("TS Timescale") .batchSize(tsBatchSize) @@ -89,14 +88,60 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements @PreDestroy protected void destroy() { - super.destroy(); if (tsQueue != null) { tsQueue.destroy(); } } @Override - protected ListenableFuture> findAllAsync(EntityId entityId, ReadTsKvQuery query) { + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { + return processFindAllAsync(tenantId, entityId, queries); + } + + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + String strKey = tsKvEntry.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); + entity.setEntityId(entityId.getId()); + entity.setTs(tsKvEntry.getTs()); + entity.setKey(keyId); + entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); + entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); + entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); + entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); + + log.trace("Saving entity to timescale db: {}", entity); + return tsQueue.add(entity); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + return Futures.immediateFuture(null); + } + + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + String strKey = query.getKey(); + Integer keyId = getOrSaveKeyId(strKey); + return service.submit(() -> { + tsKvRepository.delete( + entityId.getId(), + keyId, + query.getStartTs(), + query.getEndTs()); + return null; + }); + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + return service.submit(() -> null); + } + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(entityId, query); } else { @@ -108,8 +153,7 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } } - @Override - protected ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + private ListenableFuture> findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { String strKey = query.getKey(); Integer keyId = getOrSaveKeyId(strKey); List timescaleTsKvEntities = tsKvRepository.findAllWithLimit( @@ -153,73 +197,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements }, MoreExecutors.directExecutor()); } - @Override - public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { - return processFindAllAsync(tenantId, entityId, queries); - } - - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - return getFindLatestFuture(entityId, key); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - return getFindAllLatestFuture(entityId); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - String strKey = tsKvEntry.getKey(); - Integer keyId = getOrSaveKeyId(strKey); - TimescaleTsKvEntity entity = new TimescaleTsKvEntity(); - entity.setEntityId(entityId.getId()); - entity.setTs(tsKvEntry.getTs()); - entity.setKey(keyId); - entity.setStrValue(tsKvEntry.getStrValue().orElse(null)); - entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); - entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); - entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); - entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); - - log.trace("Saving entity to timescale db: {}", entity); - return tsQueue.add(entity); - } - - @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - return Futures.immediateFuture(null); - } - - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - return getSaveLatestFuture(entityId, tsKvEntry); - } - - @Override - public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - String strKey = query.getKey(); - Integer keyId = getOrSaveKeyId(strKey); - return service.submit(() -> { - tsKvRepository.delete( - entityId.getId(), - keyId, - query.getStartTs(), - query.getEndTs()); - return null; - }); - } - - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return getRemoveLatestFuture(entityId, query); - } - - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> null); - } - private CompletableFuture> switchAggregation(String key, long startTs, long endTs, long timeBucket, Aggregation aggregation, UUID entityId) { switch (aggregation) { case AVG: diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java index d4e80dc1f5..c6fb3574a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TsKvTimescaleRepository.java @@ -23,12 +23,12 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvCompositeKey; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.util.TimescaleDBTsDao; +import org.thingsboard.server.dao.util.TimescaleDBTsOrTsLatestDao; import java.util.List; import java.util.UUID; -@TimescaleDBTsDao +@TimescaleDBTsOrTsLatestDao public interface TsKvTimescaleRepository extends CrudRepository { @Query("SELECT tskv FROM TimescaleTsKvEntity tskv WHERE tskv.entityId = :entityId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java index 5b446129a8..881de11fa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/ts/TsKvRepository.java @@ -24,13 +24,11 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sqlts.ts.TsKvCompositeKey; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; -import org.thingsboard.server.dao.util.SqlDao; import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; -@SqlDao public interface TsKvRepository extends CrudRepository { @Query("SELECT tskv FROM TsKvEntity tskv WHERE tskv.entityId = :entityId " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java new file mode 100644 index 0000000000..3916a2fd10 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/AbstractCassandraBaseTimeseriesDao.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.datastax.oss.driver.api.core.cql.Row; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public abstract class AbstractCassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao { + public static final String DESC_ORDER = "DESC"; + public static final String GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID = "Generated query [{}] for entityType {} and entityId {}"; + public static final String INSERT_INTO = "INSERT INTO "; + public static final String SELECT_PREFIX = "SELECT "; + public static final String EQUALS_PARAM = " = ? "; + + public static KvEntry toKvEntry(Row row, String key) { + KvEntry kvEntry = null; + String strV = row.get(ModelConstants.STRING_VALUE_COLUMN, String.class); + if (strV != null) { + kvEntry = new StringDataEntry(key, strV); + } else { + Long longV = row.get(ModelConstants.LONG_VALUE_COLUMN, Long.class); + if (longV != null) { + kvEntry = new LongDataEntry(key, longV); + } else { + Double doubleV = row.get(ModelConstants.DOUBLE_VALUE_COLUMN, Double.class); + if (doubleV != null) { + kvEntry = new DoubleDataEntry(key, doubleV); + } else { + Boolean boolV = row.get(ModelConstants.BOOLEAN_VALUE_COLUMN, Boolean.class); + if (boolV != null) { + kvEntry = new BooleanDataEntry(key, boolV); + } else { + String jsonV = row.get(ModelConstants.JSON_VALUE_COLUMN, String.class); + if (StringUtils.isNoneEmpty(jsonV)) { + kvEntry = new JsonDataEntry(key, jsonV); + } else { + log.warn("All values in key-value row are nullable "); + } + } + } + } + } + return kvEntry; + } + + protected List convertResultToTsKvEntryList(List rows) { + List entries = new ArrayList<>(rows.size()); + if (!rows.isEmpty()) { + rows.forEach(row -> entries.add(convertResultToTsKvEntry(row))); + } + return entries; + } + + private TsKvEntry convertResultToTsKvEntry(Row row) { + String key = row.getString(ModelConstants.KEY_COLUMN); + long ts = row.getLong(ModelConstants.TS_COLUMN); + return new BasicTsKvEntry(ts, toKvEntry(row, key)); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index 376ea6499e..b78393b0a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -59,6 +59,9 @@ public class BaseTimeseriesService implements TimeseriesService { @Autowired private TimeseriesDao timeseriesDao; + @Autowired + private TimeseriesLatestDao timeseriesLatestDao; + @Autowired private EntityViewService entityViewService; @@ -103,7 +106,7 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.immediateFuture(new ArrayList<>()); } } - keys.forEach(key -> futures.add(timeseriesDao.findLatest(tenantId, entityId, key))); + keys.forEach(key -> futures.add(timeseriesLatestDao.findLatest(tenantId, entityId, key))); return Futures.allAsList(futures); } @@ -119,7 +122,7 @@ public class BaseTimeseriesService implements TimeseriesService { return Futures.immediateFuture(new ArrayList<>()); } } else { - return timeseriesDao.findAllLatest(tenantId, entityId); + return timeseriesLatestDao.findAllLatest(tenantId, entityId); } } @@ -151,7 +154,7 @@ public class BaseTimeseriesService implements TimeseriesService { throw new IncorrectParameterException("Telemetry data can't be stored for entity view. Read only"); } futures.add(timeseriesDao.savePartition(tenantId, entityId, tsKvEntry.getTs(), tsKvEntry.getKey(), ttl)); - futures.add(timeseriesDao.saveLatest(tenantId, entityId, tsKvEntry)); + futures.add(timeseriesLatestDao.saveLatest(tenantId, entityId, tsKvEntry)); futures.add(timeseriesDao.save(tenantId, entityId, tsKvEntry, ttl)); } @@ -187,7 +190,7 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(timeseriesDao.remove(tenantId, entityId, query)); - futures.add(timeseriesDao.removeLatest(tenantId, entityId, query)); + futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); futures.add(timeseriesDao.removePartition(tenantId, entityId, query)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 3eeff89a90..0dcef7c289 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -20,7 +20,6 @@ import com.datastax.oss.driver.api.core.cql.BoundStatement; import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.Row; -import com.datastax.oss.driver.api.core.cql.Statement; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.select.Select; import com.google.common.base.Function; @@ -30,7 +29,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; @@ -39,21 +37,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; -import org.thingsboard.server.common.data.kv.BasicTsKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; -import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.nosql.CassandraAbstractAsyncDao; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsDao; import javax.annotation.Nullable; @@ -68,7 +60,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -79,16 +70,11 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @Component @Slf4j @NoSqlTsDao -public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implements TimeseriesDao { +public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesDao implements TimeseriesDao, AggregationTimeseriesDao { - private static final int MIN_AGGREGATION_STEP_MS = 1000; - public static final String INSERT_INTO = "INSERT INTO "; - public static final String GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID = "Generated query [{}] for entityType {} and entityId {}"; - public static final String SELECT_PREFIX = "SELECT "; - public static final String EQUALS_PARAM = " = ? "; + protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; - public static final String DESC_ORDER = "DESC"; - private static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); + protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); @Autowired private Environment environment; @@ -106,13 +92,10 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem private PreparedStatement partitionInsertStmt; private PreparedStatement partitionInsertTtlStmt; - private PreparedStatement latestInsertStmt; private PreparedStatement[] saveStmts; private PreparedStatement[] saveTtlStmts; private PreparedStatement[] fetchStmtsAsc; private PreparedStatement[] fetchStmtsDesc; - private PreparedStatement findLatestStmt; - private PreparedStatement findAllLatestStmt; private PreparedStatement deleteStmt; private PreparedStatement deletePartitionStmt; @@ -157,8 +140,113 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem }, readResultsProcessingExecutor); } + @Override + public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { + List> futures = new ArrayList<>(); + ttl = computeTtl(ttl); + long partition = toPartitionTs(tsKvEntry.getTs()); + DataType type = tsKvEntry.getDataType(); + if (setNullValuesEnabled) { + processSetNullValues(tenantId, entityId, tsKvEntry, ttl, futures, partition, type); + } + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setString(2, tsKvEntry.getKey()) + .setLong(3, partition) + .setLong(4, tsKvEntry.getTs()); + addValue(tsKvEntry, stmtBuilder, 5); + if (ttl > 0) { + stmtBuilder.setInt(6, (int) ttl); + } + BoundStatement stmt = stmtBuilder.build(); + futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); + return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { + if (isFixedPartitioning()) { + return Futures.immediateFuture(null); + } + ttl = computeTtl(ttl); + long partition = toPartitionTs(tsKvEntryTs); + log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key); + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setLong(2, partition) + .setString(3, key); + if (ttl > 0) { + stmtBuilder.setInt(4, (int) ttl); + } + BoundStatement stmt = stmtBuilder.build(); + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + } - private ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long minPartition = toPartitionTs(query.getStartTs()); + long maxPartition = toPartitionTs(query.getEndTs()); + + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); + + Futures.addCallback(partitionsListFuture, new FutureCallback>() { + @Override + public void onSuccess(@Nullable List partitions) { + QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); + deleteAsync(tenantId, cursor, resultFuture); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); + } + }, readResultsProcessingExecutor); + return resultFuture; + } + + @Override + public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long minPartition = toPartitionTs(query.getStartTs()); + long maxPartition = toPartitionTs(query.getEndTs()); + if (minPartition == maxPartition) { + return Futures.immediateFuture(null); + } else { + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); + + Futures.addCallback(partitionsListFuture, new FutureCallback>() { + @Override + public void onSuccess(@Nullable List partitions) { + int index = 0; + if (minPartition != query.getStartTs()) { + index = 1; + } + List partitionsToDelete = new ArrayList<>(); + for (int i = index; i < partitions.size() - 1; i++) { + partitionsToDelete.add(partitions.get(i)); + } + QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitionsToDelete); + deletePartitionAsync(tenantId, cursor, resultFuture); + } + + @Override + public void onFailure(Throwable t) { + log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); + } + }, readResultsProcessingExecutor); + return resultFuture; + } + } + + @Override + public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { @@ -183,18 +271,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem } } - public boolean isFixedPartitioning() { - return tsFormat.getTruncateUnit().equals(ChronoUnit.FOREVER); - } - - private ListenableFuture> getPartitionsFuture(TenantId tenantId, ReadTsKvQuery query, EntityId entityId, long minPartition, long maxPartition) { - if (isFixedPartitioning()) { //no need to fetch partitions from DB - return Futures.immediateFuture(FIXED_PARTITION); - } - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - } - private ListenableFuture> findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { long minPartition = toPartitionTs(query.getStartTs()); long maxPartition = toPartitionTs(query.getEndTs()); @@ -287,10 +363,18 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem private AsyncFunction> getPartitionsArrayFunction() { return rs -> - Futures.transform(rs.allRows(readResultsProcessingExecutor), rows -> - rows.stream() - .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList()), - readResultsProcessingExecutor); + Futures.transform(rs.allRows(readResultsProcessingExecutor), rows -> + rows.stream() + .map(row -> row.getLong(ModelConstants.PARTITION_COLUMN)).collect(Collectors.toList()), + readResultsProcessingExecutor); + } + + private ListenableFuture> getPartitionsFuture(TenantId tenantId, ReadTsKvQuery query, EntityId entityId, long minPartition, long maxPartition) { + if (isFixedPartitioning()) { //no need to fetch partitions from DB + return Futures.immediateFuture(FIXED_PARTITION); + } + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); } private AsyncFunction, List> getFetchChunksAsyncFunction(TenantId tenantId, EntityId entityId, String key, Aggregation aggregation, long startTs, long endTs) { @@ -319,49 +403,8 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem }; } - @Override - public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()); - stmtBuilder.setUuid(1, entityId.getId()); - stmtBuilder.setString(2, key); - BoundStatement stmt = stmtBuilder.build(); - log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); - return getFuture(executeAsyncRead(tenantId, stmt), rs -> convertResultToTsKvEntry(key, rs.one())); - } - - @Override - public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindAllLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()); - stmtBuilder.setUuid(1, entityId.getId()); - BoundStatement stmt = stmtBuilder.build(); - log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); - return getFutureAsync(executeAsyncRead(tenantId, stmt), rs -> convertAsyncResultSetToTsKvEntryList(rs)); - } - - @Override - public ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl) { - List> futures = new ArrayList<>(); - ttl = computeTtl(ttl); - long partition = toPartitionTs(tsKvEntry.getTs()); - DataType type = tsKvEntry.getDataType(); - if (setNullValuesEnabled) { - processSetNullValues(tenantId, entityId, tsKvEntry, ttl, futures, partition, type); - } - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getSaveStmt(type) : getSaveTtlStmt(type)).bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setString(2, tsKvEntry.getKey()) - .setLong(3, partition) - .setLong(4, tsKvEntry.getTs()); - addValue(tsKvEntry, stmtBuilder, 5); - if (ttl > 0) { - stmtBuilder.setInt(6, (int) ttl); - } - BoundStatement stmt = stmtBuilder.build(); - futures.add(getFuture(executeAsyncWrite(tenantId, stmt), rs -> null)); - return Futures.transform(Futures.allAsList(futures), result -> null, MoreExecutors.directExecutor()); + private boolean isFixedPartitioning() { + return tsFormat.getTruncateUnit().equals(ChronoUnit.FOREVER); } private void processSetNullValues(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl, List> futures, long partition, DataType type) { @@ -414,26 +457,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); } - @Override - public ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl) { - if (isFixedPartitioning()) { - return Futures.immediateFuture(null); - } - ttl = computeTtl(ttl); - long partition = toPartitionTs(tsKvEntryTs); - log.debug("Saving partition {} for the entity [{}-{}] and key {}", partition, entityId.getEntityType(), entityId.getId(), key); - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder((ttl == 0 ? getPartitionInsertStmt() : getPartitionInsertTtlStmt()).bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setLong(2, partition) - .setString(3, key); - if (ttl > 0) { - stmtBuilder.setInt(4, (int) ttl); - } - BoundStatement stmt = stmtBuilder.build(); - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); - } - private long computeTtl(long ttl) { if (systemTtl > 0) { if (ttl == 0) { @@ -445,53 +468,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return ttl; } - @Override - public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { - BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); - stmtBuilder.setString(0, entityId.getEntityType().name()) - .setUuid(1, entityId.getId()) - .setString(2, tsKvEntry.getKey()) - .setLong(3, tsKvEntry.getTs()) - .set(4, tsKvEntry.getBooleanValue().orElse(null), Boolean.class) - .set(5, tsKvEntry.getStrValue().orElse(null), String.class) - .set(6, tsKvEntry.getLongValue().orElse(null), Long.class) - .set(7, tsKvEntry.getDoubleValue().orElse(null), Double.class); - Optional jsonV = tsKvEntry.getJsonValue(); - if (jsonV.isPresent()) { - stmtBuilder.setString(8, tsKvEntry.getJsonValue().get()); - } else { - stmtBuilder.setToNull(8); - } - BoundStatement stmt = stmtBuilder.build(); - - return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); - } - - @Override - public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - - Futures.addCallback(partitionsListFuture, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List partitions) { - QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); - deleteAsync(tenantId, cursor, resultFuture); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); - } - }, readResultsProcessingExecutor); - return resultFuture; - } - private void deleteAsync(TenantId tenantId, final QueryCursor cursor, final SimpleListenableFuture resultFuture) { if (!cursor.hasNextPartition()) { resultFuture.set(null); @@ -534,119 +510,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return deleteStmt; } - @Override - public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - ListenableFuture latestEntryFuture = findLatest(tenantId, entityId, query.getKey()); - - ListenableFuture booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> { - long ts = latestEntry.getTs(); - if (ts > query.getStartTs() && ts <= query.getEndTs()) { - return true; - } else { - log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey()); - } - return false; - }, readResultsProcessingExecutor); - - ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return deleteLatest(tenantId, entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - Futures.addCallback(removedLatestFuture, new FutureCallback() { - @Override - public void onSuccess(@Nullable Void result) { - if (query.getRewriteLatestIfDeleted()) { - ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { - if (isRemove) { - return getNewLatestEntryFuture(tenantId, entityId, query); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - - try { - resultFuture.set(savedLatestFuture.get()); - } catch (InterruptedException | ExecutionException e) { - log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); - } - } else { - resultFuture.set(null); - } - } - - @Override - public void onFailure(Throwable t) { - log.warn("[{}] Failed to process remove of the latest value", entityId, t); - } - }, MoreExecutors.directExecutor()); - return resultFuture; - } - - private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long startTs = 0; - long endTs = query.getStartTs() - 1; - ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, - Aggregation.NONE, DESC_ORDER); - ListenableFuture> future = findAllAsync(tenantId, entityId, findNewLatestQuery); - - return Futures.transformAsync(future, entryList -> { - if (entryList.size() == 1) { - return saveLatest(tenantId, entityId, entryList.get(0)); - } else { - log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); - } - return Futures.immediateFuture(null); - }, readResultsProcessingExecutor); - } - - private ListenableFuture deleteLatest(TenantId tenantId, EntityId entityId, String key) { - Statement delete = QueryBuilder.deleteFrom(ModelConstants.TS_KV_LATEST_CF) - .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) - .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) - .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)).build(); - log.debug("Remove request: {}", delete.toString()); - return getFuture(executeAsyncWrite(tenantId, delete), rs -> null); - } - - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - if (minPartition == maxPartition) { - return Futures.immediateFuture(null); - } else { - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - - Futures.addCallback(partitionsListFuture, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List partitions) { - int index = 0; - if (minPartition != query.getStartTs()) { - index = 1; - } - List partitionsToDelete = new ArrayList<>(); - for (int i = index; i < partitions.size() - 1; i++) { - partitionsToDelete.add(partitions.get(i)); - } - QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitionsToDelete); - deletePartitionAsync(tenantId, cursor, resultFuture); - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); - } - }, readResultsProcessingExecutor); - return resultFuture; - } - } - private void deletePartitionAsync(TenantId tenantId, final QueryCursor cursor, final SimpleListenableFuture resultFuture) { if (!cursor.hasNextPartition()) { resultFuture.set(null); @@ -685,79 +548,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return deletePartitionStmt; } - private ListenableFuture> convertAsyncResultSetToTsKvEntryList(TbResultSet rs) { - return Futures.transform(rs.allRows(readResultsProcessingExecutor), - rows -> this.convertResultToTsKvEntryList(rows), readResultsProcessingExecutor); - } - - private List convertResultToTsKvEntryList(List rows) { - List entries = new ArrayList<>(rows.size()); - if (!rows.isEmpty()) { - rows.forEach(row -> entries.add(convertResultToTsKvEntry(row))); - } - return entries; - } - - private TsKvEntry convertResultToTsKvEntry(String key, Row row) { - if (row != null) { - long ts = row.getLong(ModelConstants.TS_COLUMN); - return new BasicTsKvEntry(ts, toKvEntry(row, key)); - } else { - return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); - } - } - - private TsKvEntry convertResultToTsKvEntry(Row row) { - String key = row.getString(ModelConstants.KEY_COLUMN); - long ts = row.getLong(ModelConstants.TS_COLUMN); - return new BasicTsKvEntry(ts, toKvEntry(row, key)); - } - - public static KvEntry toKvEntry(Row row, String key) { - KvEntry kvEntry = null; - String strV = row.get(ModelConstants.STRING_VALUE_COLUMN, String.class); - if (strV != null) { - kvEntry = new StringDataEntry(key, strV); - } else { - Long longV = row.get(ModelConstants.LONG_VALUE_COLUMN, Long.class); - if (longV != null) { - kvEntry = new LongDataEntry(key, longV); - } else { - Double doubleV = row.get(ModelConstants.DOUBLE_VALUE_COLUMN, Double.class); - if (doubleV != null) { - kvEntry = new DoubleDataEntry(key, doubleV); - } else { - Boolean boolV = row.get(ModelConstants.BOOLEAN_VALUE_COLUMN, Boolean.class); - if (boolV != null) { - kvEntry = new BooleanDataEntry(key, boolV); - } else { - String jsonV = row.get(ModelConstants.JSON_VALUE_COLUMN, String.class); - if (StringUtils.isNoneEmpty(jsonV)) { - kvEntry = new JsonDataEntry(key, jsonV); - } else { - log.warn("All values in key-value row are nullable "); - } - } - } - } - } - return kvEntry; - } - - /** - * Select existing partitions from the table - * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity - */ - private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { - Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) - .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) - .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) - .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)) - .whereColumn(ModelConstants.PARTITION_COLUMN).isGreaterThanOrEqualTo(literal(minPartition)) - .whereColumn(ModelConstants.PARTITION_COLUMN).isLessThanOrEqualTo(literal(maxPartition)); - return executeAsyncRead(tenantId, select.build()); - } - private PreparedStatement getSaveStmt(DataType dataType) { if (saveStmts == null) { saveStmts = new PreparedStatement[DataType.values().length]; @@ -792,63 +582,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return saveTtlStmts[dataType.ordinal()]; } - private PreparedStatement getFetchStmt(Aggregation aggType, String orderBy) { - switch (orderBy) { - case ASC_ORDER: - if (fetchStmtsAsc == null) { - fetchStmtsAsc = initFetchStmt(orderBy); - } - return fetchStmtsAsc[aggType.ordinal()]; - case DESC_ORDER: - if (fetchStmtsDesc == null) { - fetchStmtsDesc = initFetchStmt(orderBy); - } - return fetchStmtsDesc[aggType.ordinal()]; - default: - throw new RuntimeException("Not supported" + orderBy + "order!"); - } - } - - private PreparedStatement[] initFetchStmt(String orderBy) { - PreparedStatement[] fetchStmts = new PreparedStatement[Aggregation.values().length]; - for (Aggregation type : Aggregation.values()) { - if (type == Aggregation.SUM && fetchStmts[Aggregation.AVG.ordinal()] != null) { - fetchStmts[type.ordinal()] = fetchStmts[Aggregation.AVG.ordinal()]; - } else if (type == Aggregation.AVG && fetchStmts[Aggregation.SUM.ordinal()] != null) { - fetchStmts[type.ordinal()] = fetchStmts[Aggregation.SUM.ordinal()]; - } else { - fetchStmts[type.ordinal()] = prepare(SELECT_PREFIX + - String.join(", ", ModelConstants.getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF - + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM - + "AND " + ModelConstants.TS_COLUMN + " > ? " - + "AND " + ModelConstants.TS_COLUMN + " <= ?" - + (type == Aggregation.NONE ? " ORDER BY " + ModelConstants.TS_COLUMN + " " + orderBy + " LIMIT ?" : "")); - } - } - return fetchStmts; - } - - private PreparedStatement getLatestStmt() { - if (latestInsertStmt == null) { - latestInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_LATEST_CF + - "(" + ModelConstants.ENTITY_TYPE_COLUMN + - "," + ModelConstants.ENTITY_ID_COLUMN + - "," + ModelConstants.KEY_COLUMN + - "," + ModelConstants.TS_COLUMN + - "," + ModelConstants.BOOLEAN_VALUE_COLUMN + - "," + ModelConstants.STRING_VALUE_COLUMN + - "," + ModelConstants.LONG_VALUE_COLUMN + - "," + ModelConstants.DOUBLE_VALUE_COLUMN + - "," + ModelConstants.JSON_VALUE_COLUMN + ")" + - " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); - } - return latestInsertStmt; - } - - private PreparedStatement getPartitionInsertStmt() { if (partitionInsertStmt == null) { partitionInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_PARTITIONS_CF + @@ -873,42 +606,6 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem return partitionInsertTtlStmt; } - - private PreparedStatement getFindLatestStmt() { - if (findLatestStmt == null) { - findLatestStmt = prepare(SELECT_PREFIX + - ModelConstants.KEY_COLUMN + "," + - ModelConstants.TS_COLUMN + "," + - ModelConstants.STRING_VALUE_COLUMN + "," + - ModelConstants.BOOLEAN_VALUE_COLUMN + "," + - ModelConstants.LONG_VALUE_COLUMN + "," + - ModelConstants.DOUBLE_VALUE_COLUMN + "," + - ModelConstants.JSON_VALUE_COLUMN + " " + - "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + - "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM); - } - return findLatestStmt; - } - - private PreparedStatement getFindAllLatestStmt() { - if (findAllLatestStmt == null) { - findAllLatestStmt = prepare(SELECT_PREFIX + - ModelConstants.KEY_COLUMN + "," + - ModelConstants.TS_COLUMN + "," + - ModelConstants.STRING_VALUE_COLUMN + "," + - ModelConstants.BOOLEAN_VALUE_COLUMN + "," + - ModelConstants.LONG_VALUE_COLUMN + "," + - ModelConstants.DOUBLE_VALUE_COLUMN + "," + - ModelConstants.JSON_VALUE_COLUMN + " " + - "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + - "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + - "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM); - } - return findAllLatestStmt; - } - private static String getColumnName(DataType type) { switch (type) { case BOOLEAN: @@ -951,4 +648,56 @@ public class CassandraBaseTimeseriesDao extends CassandraAbstractAsyncDao implem } } + /** + // * Select existing partitions from the table + // * {@link ModelConstants#TS_KV_PARTITIONS_CF} for the given entity + // */ + private TbResultSetFuture fetchPartitions(TenantId tenantId, EntityId entityId, String key, long minPartition, long maxPartition) { + Select select = QueryBuilder.selectFrom(ModelConstants.TS_KV_PARTITIONS_CF).column(ModelConstants.PARTITION_COLUMN) + .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) + .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) + .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)) + .whereColumn(ModelConstants.PARTITION_COLUMN).isGreaterThanOrEqualTo(literal(minPartition)) + .whereColumn(ModelConstants.PARTITION_COLUMN).isLessThanOrEqualTo(literal(maxPartition)); + return executeAsyncRead(tenantId, select.build()); + } + + private PreparedStatement getFetchStmt(Aggregation aggType, String orderBy) { + switch (orderBy) { + case ASC_ORDER: + if (fetchStmtsAsc == null) { + fetchStmtsAsc = initFetchStmt(orderBy); + } + return fetchStmtsAsc[aggType.ordinal()]; + case DESC_ORDER: + if (fetchStmtsDesc == null) { + fetchStmtsDesc = initFetchStmt(orderBy); + } + return fetchStmtsDesc[aggType.ordinal()]; + default: + throw new RuntimeException("Not supported" + orderBy + "order!"); + } + } + + private PreparedStatement[] initFetchStmt(String orderBy) { + PreparedStatement[] fetchStmts = new PreparedStatement[Aggregation.values().length]; + for (Aggregation type : Aggregation.values()) { + if (type == Aggregation.SUM && fetchStmts[Aggregation.AVG.ordinal()] != null) { + fetchStmts[type.ordinal()] = fetchStmts[Aggregation.AVG.ordinal()]; + } else if (type == Aggregation.AVG && fetchStmts[Aggregation.SUM.ordinal()] != null) { + fetchStmts[type.ordinal()] = fetchStmts[Aggregation.SUM.ordinal()]; + } else { + fetchStmts[type.ordinal()] = prepare(SELECT_PREFIX + + String.join(", ", ModelConstants.getFetchColumnNames(type)) + " FROM " + ModelConstants.TS_KV_CF + + " WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.PARTITION_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.TS_COLUMN + " > ? " + + "AND " + ModelConstants.TS_COLUMN + " <= ?" + + (type == Aggregation.NONE ? " ORDER BY " + ModelConstants.TS_COLUMN + " " + orderBy + " LIMIT ?" : "")); + } + } + return fetchStmts; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java new file mode 100644 index 0000000000..00eb44c7a2 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -0,0 +1,249 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.Statement; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.nosql.TbResultSet; +import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; +import org.thingsboard.server.dao.util.NoSqlTsLatestDao; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.ExecutionException; + +import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; + +@Component +@Slf4j +@NoSqlTsLatestDao +public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimeseriesDao implements TimeseriesLatestDao { + + @Autowired + protected AggregationTimeseriesDao aggregationTimeseriesDao; + + private PreparedStatement latestInsertStmt; + private PreparedStatement findLatestStmt; + private PreparedStatement findAllLatestStmt; + + @Override + public ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()); + stmtBuilder.setUuid(1, entityId.getId()); + stmtBuilder.setString(2, key); + BoundStatement stmt = stmtBuilder.build(); + log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); + return getFuture(executeAsyncRead(tenantId, stmt), rs -> convertResultToTsKvEntry(key, rs.one())); + } + + @Override + public ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getFindAllLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()); + stmtBuilder.setUuid(1, entityId.getId()); + BoundStatement stmt = stmtBuilder.build(); + log.debug(GENERATED_QUERY_FOR_ENTITY_TYPE_AND_ENTITY_ID, stmt, entityId.getEntityType(), entityId.getId()); + return getFutureAsync(executeAsyncRead(tenantId, stmt), rs -> convertAsyncResultSetToTsKvEntryList(rs)); + } + + @Override + public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { + BoundStatementBuilder stmtBuilder = new BoundStatementBuilder(getLatestStmt().bind()); + stmtBuilder.setString(0, entityId.getEntityType().name()) + .setUuid(1, entityId.getId()) + .setString(2, tsKvEntry.getKey()) + .setLong(3, tsKvEntry.getTs()) + .set(4, tsKvEntry.getBooleanValue().orElse(null), Boolean.class) + .set(5, tsKvEntry.getStrValue().orElse(null), String.class) + .set(6, tsKvEntry.getLongValue().orElse(null), Long.class) + .set(7, tsKvEntry.getDoubleValue().orElse(null), Double.class); + Optional jsonV = tsKvEntry.getJsonValue(); + if (jsonV.isPresent()) { + stmtBuilder.setString(8, tsKvEntry.getJsonValue().get()); + } else { + stmtBuilder.setToNull(8); + } + BoundStatement stmt = stmtBuilder.build(); + + return getFuture(executeAsyncWrite(tenantId, stmt), rs -> null); + } + + @Override + public ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + ListenableFuture latestEntryFuture = findLatest(tenantId, entityId, query.getKey()); + + ListenableFuture booleanFuture = Futures.transform(latestEntryFuture, latestEntry -> { + long ts = latestEntry.getTs(); + if (ts > query.getStartTs() && ts <= query.getEndTs()) { + return true; + } else { + log.trace("Won't be deleted latest value for [{}], key - {}", entityId, query.getKey()); + } + return false; + }, readResultsProcessingExecutor); + + ListenableFuture removedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return deleteLatest(tenantId, entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + + final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); + Futures.addCallback(removedLatestFuture, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void result) { + if (query.getRewriteLatestIfDeleted()) { + ListenableFuture savedLatestFuture = Futures.transformAsync(booleanFuture, isRemove -> { + if (isRemove) { + return getNewLatestEntryFuture(tenantId, entityId, query); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + + try { + resultFuture.set(savedLatestFuture.get()); + } catch (InterruptedException | ExecutionException e) { + log.warn("Could not get latest saved value for [{}], {}", entityId, query.getKey(), e); + } + } else { + resultFuture.set(null); + } + } + + @Override + public void onFailure(Throwable t) { + log.warn("[{}] Failed to process remove of the latest value", entityId, t); + } + }, MoreExecutors.directExecutor()); + return resultFuture; + } + + private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { + long startTs = 0; + long endTs = query.getStartTs() - 1; + ReadTsKvQuery findNewLatestQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, + Aggregation.NONE, DESC_ORDER); + ListenableFuture> future = aggregationTimeseriesDao.findAllAsync(tenantId, entityId, findNewLatestQuery); + + return Futures.transformAsync(future, entryList -> { + if (entryList.size() == 1) { + return saveLatest(tenantId, entityId, entryList.get(0)); + } else { + log.trace("Could not find new latest value for [{}], key - {}", entityId, query.getKey()); + } + return Futures.immediateFuture(null); + }, readResultsProcessingExecutor); + } + + private ListenableFuture deleteLatest(TenantId tenantId, EntityId entityId, String key) { + Statement delete = QueryBuilder.deleteFrom(ModelConstants.TS_KV_LATEST_CF) + .whereColumn(ModelConstants.ENTITY_TYPE_COLUMN).isEqualTo(literal(entityId.getEntityType().name())) + .whereColumn(ModelConstants.ENTITY_ID_COLUMN).isEqualTo(literal(entityId.getId())) + .whereColumn(ModelConstants.KEY_COLUMN).isEqualTo(literal(key)).build(); + log.debug("Remove request: {}", delete.toString()); + return getFuture(executeAsyncWrite(tenantId, delete), rs -> null); + } + + private ListenableFuture> convertAsyncResultSetToTsKvEntryList(TbResultSet rs) { + return Futures.transform(rs.allRows(readResultsProcessingExecutor), + rows -> this.convertResultToTsKvEntryList(rows), readResultsProcessingExecutor); + } + + private TsKvEntry convertResultToTsKvEntry(String key, Row row) { + if (row != null) { + long ts = row.getLong(ModelConstants.TS_COLUMN); + return new BasicTsKvEntry(ts, toKvEntry(row, key)); + } else { + return new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(key, null)); + } + } + + private PreparedStatement getLatestStmt() { + if (latestInsertStmt == null) { + latestInsertStmt = prepare(INSERT_INTO + ModelConstants.TS_KV_LATEST_CF + + "(" + ModelConstants.ENTITY_TYPE_COLUMN + + "," + ModelConstants.ENTITY_ID_COLUMN + + "," + ModelConstants.KEY_COLUMN + + "," + ModelConstants.TS_COLUMN + + "," + ModelConstants.BOOLEAN_VALUE_COLUMN + + "," + ModelConstants.STRING_VALUE_COLUMN + + "," + ModelConstants.LONG_VALUE_COLUMN + + "," + ModelConstants.DOUBLE_VALUE_COLUMN + + "," + ModelConstants.JSON_VALUE_COLUMN + ")" + + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"); + } + return latestInsertStmt; + } + + private PreparedStatement getFindLatestStmt() { + if (findLatestStmt == null) { + findLatestStmt = prepare(SELECT_PREFIX + + ModelConstants.KEY_COLUMN + "," + + ModelConstants.TS_COLUMN + "," + + ModelConstants.STRING_VALUE_COLUMN + "," + + ModelConstants.BOOLEAN_VALUE_COLUMN + "," + + ModelConstants.LONG_VALUE_COLUMN + "," + + ModelConstants.DOUBLE_VALUE_COLUMN + "," + + ModelConstants.JSON_VALUE_COLUMN + " " + + "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + + "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.KEY_COLUMN + EQUALS_PARAM); + } + return findLatestStmt; + } + + private PreparedStatement getFindAllLatestStmt() { + if (findAllLatestStmt == null) { + findAllLatestStmt = prepare(SELECT_PREFIX + + ModelConstants.KEY_COLUMN + "," + + ModelConstants.TS_COLUMN + "," + + ModelConstants.STRING_VALUE_COLUMN + "," + + ModelConstants.BOOLEAN_VALUE_COLUMN + "," + + ModelConstants.LONG_VALUE_COLUMN + "," + + ModelConstants.DOUBLE_VALUE_COLUMN + "," + + ModelConstants.JSON_VALUE_COLUMN + " " + + "FROM " + ModelConstants.TS_KV_LATEST_CF + " " + + "WHERE " + ModelConstants.ENTITY_TYPE_COLUMN + EQUALS_PARAM + + "AND " + ModelConstants.ENTITY_ID_COLUMN + EQUALS_PARAM); + } + return findAllLatestStmt; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index 39d965630e..dda2b227c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -31,19 +31,11 @@ public interface TimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); - ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key); - - ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); - ListenableFuture save(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry, long ttl); ListenableFuture savePartition(TenantId tenantId, EntityId entityId, long tsKvEntryTs, String key, long ttl); - ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); - ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java new file mode 100644 index 0000000000..7392e0e3d4 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.List; + +public interface TimeseriesLatestDao { + + ListenableFuture findLatest(TenantId tenantId, EntityId entityId, String key); + + ListenableFuture> findAllLatest(TenantId tenantId, EntityId entityId); + + ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry); + + ListenableFuture removeLatest(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 4da1fb7028..314684c3fe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -113,7 +113,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic public User saveUser(User user) { log.trace("Executing saveUser [{}]", user); userValidator.validate(user, User::getTenantId); - if (user.getId() == null && !userLoginCaseSensitive) { + if (!userLoginCaseSensitive) { user.setEmail(user.getEmail().toLowerCase()); } User savedUser = userDao.save(user.getTenantId(), user); diff --git a/dao/src/main/resources/cassandra/schema-ts-latest.cql b/dao/src/main/resources/cassandra/schema-ts-latest.cql new file mode 100644 index 0000000000..623f530888 --- /dev/null +++ b/dao/src/main/resources/cassandra/schema-ts-latest.cql @@ -0,0 +1,34 @@ +-- +-- Copyright © 2016-2020 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE KEYSPACE IF NOT EXISTS thingsboard +WITH replication = { + 'class' : 'SimpleStrategy', + 'replication_factor' : 1 +}; + +CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( + entity_type text, // (DEVICE, CUSTOMER, TENANT) + entity_id timeuuid, + key text, + ts bigint, + bool_v boolean, + str_v text, + long_v bigint, + dbl_v double, + json_v text, + PRIMARY KEY (( entity_type, entity_id ), key) +) WITH compaction = { 'class' : 'LeveledCompactionStrategy' }; diff --git a/dao/src/main/resources/cassandra/schema-ts.cql b/dao/src/main/resources/cassandra/schema-ts.cql index c0f4b74467..957c6ee6eb 100644 --- a/dao/src/main/resources/cassandra/schema-ts.cql +++ b/dao/src/main/resources/cassandra/schema-ts.cql @@ -42,16 +42,3 @@ CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_partitions_cf ( PRIMARY KEY (( entity_type, entity_id, key ), partition) ) WITH CLUSTERING ORDER BY ( partition ASC ) AND compaction = { 'class' : 'LeveledCompactionStrategy' }; - -CREATE TABLE IF NOT EXISTS thingsboard.ts_kv_latest_cf ( - entity_type text, // (DEVICE, CUSTOMER, TENANT) - entity_id timeuuid, - key text, - ts bigint, - bool_v boolean, - str_v text, - long_v bigint, - dbl_v double, - json_v text, - PRIMARY KEY (( entity_type, entity_id ), key) -) WITH compaction = { 'class' : 'LeveledCompactionStrategy' }; diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index f634590598..664925db76 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -272,3 +272,20 @@ CREATE TABLE IF NOT EXISTS entity_view ( additional_info varchar ); +CREATE TABLE IF NOT EXISTS ts_kv_latest ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v varchar(10000000), + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( + key varchar(255) NOT NULL, + key_id int GENERATED BY DEFAULT AS IDENTITY(start with 0 increment by 1) UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index af4d5d158c..50924bbbd4 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -14,6 +14,13 @@ -- limitations under the License. -- +CREATE TABLE IF NOT EXISTS tb_schema_settings +( + schema_version bigint NOT NULL, + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) +); + +INSERT INTO tb_schema_settings (schema_version) VALUES (3001000) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 3001000; CREATE TABLE IF NOT EXISTS admin_settings ( id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, @@ -331,3 +338,4 @@ BEGIN '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); END; $$ LANGUAGE plpgsql; + diff --git a/dao/src/main/resources/sql/schema-timescale.sql b/dao/src/main/resources/sql/schema-timescale.sql index b8e62196a6..a9816f4a7a 100644 --- a/dao/src/main/resources/sql/schema-timescale.sql +++ b/dao/src/main/resources/sql/schema-timescale.sql @@ -46,14 +46,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_latest ( CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) ); -CREATE TABLE IF NOT EXISTS tb_schema_settings -( - schema_version bigint NOT NULL, - CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) -); - -INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001; - CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS $$ BEGIN diff --git a/dao/src/main/resources/sql/schema-ts-hsql.sql b/dao/src/main/resources/sql/schema-ts-hsql.sql index ce4e4d4275..eee09516fc 100644 --- a/dao/src/main/resources/sql/schema-ts-hsql.sql +++ b/dao/src/main/resources/sql/schema-ts-hsql.sql @@ -28,18 +28,6 @@ CREATE TABLE IF NOT EXISTS ts_kv ( CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ); -CREATE TABLE IF NOT EXISTS ts_kv_latest ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v varchar(10000000), - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( key varchar(255) NOT NULL, key_id int GENERATED BY DEFAULT AS IDENTITY(start with 0 increment by 1) UNIQUE, diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 0b5cab6ebb..ef6c51aa0a 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -27,19 +27,6 @@ CREATE TABLE IF NOT EXISTS ts_kv CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) ) PARTITION BY RANGE (ts); -CREATE TABLE IF NOT EXISTS ts_kv_latest -( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( key varchar(255) NOT NULL, @@ -47,14 +34,6 @@ CREATE TABLE IF NOT EXISTS ts_kv_dictionary CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) ); -CREATE TABLE IF NOT EXISTS tb_schema_settings -( - schema_version bigint NOT NULL, - CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) -); - -INSERT INTO tb_schema_settings (schema_version) VALUES (2005001) ON CONFLICT (schema_version) DO UPDATE SET schema_version = 2005001; - CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) LANGUAGE plpgsql AS $$ diff --git a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java b/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java index 3dae53bb83..f7d45ff13d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java +++ b/dao/src/test/java/org/thingsboard/server/dao/JpaDbunitTestConfig.java @@ -22,7 +22,6 @@ import org.dbunit.ext.hsqldb.HsqldbDataTypeFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.thingsboard.server.dao.util.SqlDao; import javax.sql.DataSource; import java.io.IOException; @@ -32,7 +31,6 @@ import java.sql.SQLException; * Created by Valerii Sosliuk on 5/6/2017. */ @Configuration -@SqlDao public class JpaDbunitTestConfig { @Autowired diff --git a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java index 824f5887e2..1f467209e5 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java +++ b/dao/src/test/java/org/thingsboard/server/dao/NoSqlDaoServiceTestSuite.java @@ -40,7 +40,8 @@ public class NoSqlDaoServiceTestSuite { public static CustomCassandraCQLUnit cassandraUnit = new CustomCassandraCQLUnit( Arrays.asList( - new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false) + new ClassPathCQLDataSet("cassandra/schema-ts.cql", false, false), + new ClassPathCQLDataSet("cassandra/schema-ts-latest.cql", false, false) ), "cassandra-test.yaml", 30000L); diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 8f827eab43..eefa4f0b95 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -1,4 +1,5 @@ database.ts.type=cassandra +database.ts_latest.type=cassandra sql.ts_inserts_executor_type=fixed sql.ts_inserts_fixed_thread_pool_size=10 diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index 11ca8b8daf..123c0506cc 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -1,4 +1,5 @@ database.ts.type=sql +database.ts_latest.type=sql sql.ts_inserts_executor_type=fixed sql.ts_inserts_fixed_thread_pool_size=200 diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index fedba0c821..071ed9ca08 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -25,8 +25,10 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmStatus; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -56,8 +58,13 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode processAlarm(TbContext ctx, TbMsg msg) { String alarmType = TbNodeUtils.processPattern(this.config.getAlarmType(), msg.getMetaData()); - ListenableFuture latest = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); - return Futures.transformAsync(latest, a -> { + ListenableFuture alarmFuture; + if(msg.getOriginator().getEntityType().equals(EntityType.ALARM)){ + alarmFuture = ctx.getAlarmService().findAlarmByIdAsync(ctx.getTenantId(), new AlarmId(msg.getOriginator().getId())); + } else { + alarmFuture = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); + } + return Futures.transformAsync(alarmFuture, a -> { if (a != null && !a.getStatus().isCleared()) { return clearAlarm(ctx, msg, a); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index 07a0508a87..c5739e24ff 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -16,8 +16,13 @@ package org.thingsboard.rule.engine.filter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -30,7 +35,7 @@ import org.thingsboard.server.common.msg.session.SessionMsgType; configClazz = EmptyNodeConfiguration.class, relationTypes = {"Post attributes", "Post telemetry", "RPC Request from Device", "RPC Request to Device", "Activity Event", "Inactivity Event", "Connect Event", "Disconnect Event", "Entity Created", "Entity Updated", "Entity Deleted", "Entity Assigned", - "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other"}, + "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other", "Entity Assigned From Tenant", "Entity Assigned To Tenant"}, nodeDescription = "Route incoming messages by Message Type", nodeDetails = "Sends messages with message types \"Post attributes\", \"Post telemetry\", \"RPC Request\" etc. via corresponding chain, otherwise Other chain is used.", uiResources = {"static/rulenode/rulenode-core-config.js"}, @@ -81,6 +86,10 @@ public class TbMsgTypeSwitchNode implements TbNode { relationType = "Alarm Cleared"; } else if (msg.getType().equals(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE)) { relationType = "RPC Request to Device"; + } else if (msg.getType().equals(DataConstants.ENTITY_ASSIGNED_FROM_TENANT)) { + relationType = "Entity Assigned From Tenant"; + } else if (msg.getType().equals(DataConstants.ENTITY_ASSIGNED_TO_TENANT)) { + relationType = "Entity Assigned To Tenant"; } else { relationType = "Other"; } diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 401adea879..e9d7256520 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -74,6 +74,7 @@ export interface EntityDataSubscriptionOptions { export class EntityDataSubscription { + private entityDataSubscriptionOptions = this.listener.subscriptionOptions; private datasourceType: DatasourceType = this.entityDataSubscriptionOptions.datasourceType; private history: boolean; private realtime: boolean; @@ -103,8 +104,7 @@ export class EntityDataSubscription { private dataResolved = false; private started = false; - constructor(public entityDataSubscriptionOptions: EntityDataSubscriptionOptions, - private listener: EntityDataListener, + constructor(private listener: EntityDataListener, private telemetryService: TelemetryService, private utils: UtilsService) { this.initializeSubscription(); diff --git a/ui-ngx/src/app/core/api/entity-data.service.ts b/ui-ngx/src/app/core/api/entity-data.service.ts index 22f3bfcd15..a5ef7c2c02 100644 --- a/ui-ngx/src/app/core/api/entity-data.service.ts +++ b/ui-ngx/src/app/core/api/entity-data.service.ts @@ -17,7 +17,7 @@ import { DataSetHolder, Datasource, DatasourceType, widgetType } from '@shared/models/widget.models'; import { SubscriptionTimewindow } from '@shared/models/time/time.models'; import { EntityData, EntityDataPageLink, KeyFilter } from '@shared/models/query/query.models'; -import { PageData } from '@shared/models/page/page-data'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { Injectable } from '@angular/core'; import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service'; import { UtilsService } from '@core/services/utils.service'; @@ -41,6 +41,7 @@ export interface EntityDataListener { initialPageDataChanged?: (nextPageData: PageData) => void; updateRealtimeSubscription?: () => SubscriptionTimewindow; setRealtimeSubscription?: (subscriptionTimewindow: SubscriptionTimewindow) => void; + subscriptionOptions?: EntityDataSubscriptionOptions; subscription?: EntityDataSubscription; } @@ -61,19 +62,24 @@ export class EntityDataService { public prepareSubscription(listener: EntityDataListener): Observable { const datasource = listener.configDatasource; + listener.subscriptionOptions = this.createSubscriptionOptions( + datasource, + listener.subscriptionType, + datasource.pageLink, + datasource.keyFilters, + null, + false); if (datasource.type === DatasourceType.entity && (!datasource.entityFilter || !datasource.pageLink)) { return of(null); } - listener.subscription = this.createSubscription(listener, - datasource.pageLink, datasource.keyFilters, null, - false); + listener.subscription = new EntityDataSubscription(listener, this.telemetryService, this.utils); return listener.subscription.subscribe(); } public startSubscription(listener: EntityDataListener) { if (listener.subscription) { if (listener.subscriptionType === widgetType.timeseries) { - listener.subscription.entityDataSubscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); + listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); } listener.subscription.start(); } @@ -83,13 +89,21 @@ export class EntityDataService { pageLink: EntityDataPageLink, keyFilters: KeyFilter[]): Observable { const datasource = listener.configDatasource; + listener.subscriptionOptions = this.createSubscriptionOptions( + datasource, + listener.subscriptionType, + pageLink, + datasource.keyFilters, + keyFilters, + true); if (datasource.type === DatasourceType.entity && (!datasource.entityFilter || !pageLink)) { + listener.dataLoaded(emptyPageData(), [], + listener.configDatasourceIndex, listener.subscriptionOptions.pageLink); return of(null); } - listener.subscription = this.createSubscription(listener, - pageLink, datasource.keyFilters, keyFilters,true); + listener.subscription = new EntityDataSubscription(listener, this.telemetryService, this.utils); if (listener.subscriptionType === widgetType.timeseries) { - listener.subscription.entityDataSubscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); + listener.subscriptionOptions.subscriptionTimewindow = deepClone(listener.subscriptionTimewindow); } return listener.subscription.subscribe(); } @@ -100,12 +114,12 @@ export class EntityDataService { } } - private createSubscription(listener: EntityDataListener, - pageLink: EntityDataPageLink, - keyFilters: KeyFilter[], - additionalKeyFilters: KeyFilter[], - isPaginatedDataSubscription: boolean): EntityDataSubscription { - const datasource = listener.configDatasource; + private createSubscriptionOptions(datasource: Datasource, + subscriptionType: widgetType, + pageLink: EntityDataPageLink, + keyFilters: KeyFilter[], + additionalKeyFilters: KeyFilter[], + isPaginatedDataSubscription: boolean): EntityDataSubscriptionOptions { const subscriptionDataKeys: Array = []; datasource.dataKeys.forEach((dataKey) => { const subscriptionDataKey: SubscriptionDataKey = { @@ -119,7 +133,7 @@ export class EntityDataService { const entityDataSubscriptionOptions: EntityDataSubscriptionOptions = { datasourceType: datasource.type, dataKeys: subscriptionDataKeys, - type: listener.subscriptionType + type: subscriptionType }; if (entityDataSubscriptionOptions.datasourceType === DatasourceType.entity) { entityDataSubscriptionOptions.entityFilter = datasource.entityFilter; @@ -128,8 +142,6 @@ export class EntityDataService { entityDataSubscriptionOptions.additionalKeyFilters = additionalKeyFilters; } entityDataSubscriptionOptions.isPaginatedDataSubscription = isPaginatedDataSubscription; - return new EntityDataSubscription(entityDataSubscriptionOptions, - listener, this.telemetryService, this.utils); + return entityDataSubscriptionOptions; } - } diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 47ea205db6..d664e06a5e 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -735,8 +735,12 @@ export class WidgetSubscription implements IWidgetSubscription { if (this.type === widgetType.alarm) { this.updateAlarmDataSubscription(); } else { - this.notifyDataLoading(); - this.dataSubscribe(); + if (this.hasDataPageLink) { + this.updateDataSubscriptions(); + } else { + this.notifyDataLoading(); + this.dataSubscribe(); + } } } } @@ -1017,8 +1021,8 @@ export class WidgetSubscription implements IWidgetSubscription { for (let datasourceIndex = 0; datasourceIndex < this.entityDataListeners.length; datasourceIndex++) { const entityDataListener = this.entityDataListeners[datasourceIndex]; if (entityDataListener) { - const pageLink = entityDataListener.subscription.entityDataSubscriptionOptions.pageLink; - const keyFilters = entityDataListener.subscription.entityDataSubscriptionOptions.additionalKeyFilters; + const pageLink = entityDataListener.subscriptionOptions.pageLink; + const keyFilters = entityDataListener.subscriptionOptions.additionalKeyFilters; this.subscribeForPaginatedData(datasourceIndex, pageLink, keyFilters); } } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts index c88d23fa10..c79a115f4b 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/alarms-table-widget.component.ts @@ -376,6 +376,8 @@ export class AlarmsTableWidgetComponent extends PageComponent implements OnInit, } this.stylesInfo[dataKey.def] = getCellStyleInfo(keySettings); this.contentsInfo[dataKey.def] = getCellContentInfo(keySettings, 'value, alarm, ctx'); + this.contentsInfo[dataKey.def].units = dataKey.units; + this.contentsInfo[dataKey.def].decimals = dataKey.decimals; this.columnWidth[dataKey.def] = getColumnWidth(keySettings); this.columns.push(dataKey); @@ -932,7 +934,7 @@ class AlarmsDatasource implements DataSource { } } } - alarm[dataKey.label] = value; + alarm[dataKey.name] = value; }); return alarm; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts index 77aff9c658..696cfdad42 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/entities-table-widget.component.ts @@ -87,7 +87,6 @@ import { import { sortItems } from '@shared/models/page/page-link'; import { entityFields } from '@shared/models/entity.models'; import { DatePipe } from '@angular/common'; -import { alarmFields } from '@shared/models/alarm.models'; interface EntitiesTableWidgetSettings extends TableWidgetSettings { entitiesTitle: string; @@ -659,7 +658,9 @@ class EntityDatasource implements DataSource { this.dataKeys.forEach((dataKey, index) => { const keyData = data[index].data; if (keyData && keyData.length && keyData[0].length > 1) { - entity[dataKey.label] = keyData[0][1]; + if (data[index].dataKey.type !== DataKeyType.entityField || !entity.hasOwnProperty(dataKey.label)) { + entity[dataKey.label] = keyData[0][1]; + } } else { entity[dataKey.label] = ''; } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts index 73445707bf..4adc5aacc1 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/schemes.ts @@ -14,8 +14,6 @@ /// limitations under the License. /// -import { DEFAULT_MAP_PAGE_SIZE } from '@home/components/widget/lib/maps/map-models'; - export const googleMapSettingsSchema = { schema: { @@ -247,7 +245,7 @@ export const commonMapSettingsSchema = mapPageSize: { title: 'Map page size load entities', type: 'number', - default: DEFAULT_MAP_PAGE_SIZE + default: 16384 }, defaultCenterPosition: { title: 'Default map center position (0,0)', diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts index 1ab9ea6fc4..9e079bc247 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/table-widget.models.ts @@ -189,7 +189,7 @@ export function getAlarmValue(alarm: AlarmDataInfo, key: EntityColumn) { if (alarmField) { return getDescendantProp(alarm, alarmField.value); } else { - return getDescendantProp(alarm, key.label); + return getDescendantProp(alarm, key.name); } } diff --git a/ui-ngx/src/app/shared/models/audit-log.models.ts b/ui-ngx/src/app/shared/models/audit-log.models.ts index 165b3c9d46..1b450d5568 100644 --- a/ui-ngx/src/app/shared/models/audit-log.models.ts +++ b/ui-ngx/src/app/shared/models/audit-log.models.ts @@ -49,7 +49,9 @@ export enum ActionType { ALARM_CLEAR = 'ALARM_CLEAR', LOGIN = 'LOGIN', LOGOUT = 'LOGOUT', - LOCKOUT = 'LOCKOUT' + LOCKOUT = 'LOCKOUT', + ASSIGNED_FROM_TENANT = 'ASSIGNED_FROM_TENANT', + ASSIGNED_TO_TENANT = 'ASSIGNED_TO_TENANT' } export enum ActionStatus { @@ -79,7 +81,9 @@ export const actionTypeTranslations = new Map( [ActionType.ALARM_CLEAR, 'audit-log.type-alarm-clear'], [ActionType.LOGIN, 'audit-log.type-login'], [ActionType.LOGOUT, 'audit-log.type-logout'], - [ActionType.LOCKOUT, 'audit-log.type-lockout'] + [ActionType.LOCKOUT, 'audit-log.type-lockout'], + [ActionType.ASSIGNED_FROM_TENANT, 'audit-log.type-assigned-from-tenant'], + [ActionType.ASSIGNED_TO_TENANT, 'audit-log.type-assigned-to-tenant'] ] ); diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 8ea9a8b37f..6a3c156602 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -100,6 +100,7 @@ export const HelpLinks = { ruleNodeAwsSqs: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#aws-sqs-node', ruleNodeKafka: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#kafka-node', ruleNodeMqtt: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#mqtt-node', + ruleNodeAzureIotHub: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#azure-iot-hub-node', ruleNodeRabbitMq: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#rabbitmq-node', ruleNodeRestApiCall: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#rest-api-call-node', ruleNodeSendEmail: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/external-nodes/#send-email-node', diff --git a/ui-ngx/src/app/shared/models/rule-node.models.ts b/ui-ngx/src/app/shared/models/rule-node.models.ts index d025728bd3..4d5a58f396 100644 --- a/ui-ngx/src/app/shared/models/rule-node.models.ts +++ b/ui-ngx/src/app/shared/models/rule-node.models.ts @@ -412,6 +412,7 @@ const ruleNodeClazzHelpLinkMap = { 'org.thingsboard.rule.engine.aws.sqs.TbSqsNode': 'ruleNodeAwsSqs', 'org.thingsboard.rule.engine.kafka.TbKafkaNode': 'ruleNodeKafka', 'org.thingsboard.rule.engine.mqtt.TbMqttNode': 'ruleNodeMqtt', + 'org.thingsboard.rule.engine.mqtt.azure.TbAzureIotHubNode': 'ruleNodeAzureIotHub', 'org.thingsboard.rule.engine.rabbitmq.TbRabbitMqNode': 'ruleNodeRabbitMq', 'org.thingsboard.rule.engine.rest.TbRestApiCallNode': 'ruleNodeRestApiCall', 'org.thingsboard.rule.engine.mail.TbSendEmailNode': 'ruleNodeSendEmail' diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 7ef23c9167..a5958dbd99 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -371,7 +371,9 @@ "action-data": "Action data", "failure-details": "Failure details", "search": "Search audit logs", - "clear-search": "Clear search" + "clear-search": "Clear search", + "type-assigned-from-tenant": "Assigned from Tenant", + "type-assigned-to-tenant": "Assigned to Tenant" }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?",