diff --git a/application/pom.xml b/application/pom.xml index c2400f50e5..b4211abc76 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -28,8 +28,7 @@ ThingsBoard Server Application https://thingsboard.io - Open-source IoT Platform - Device management, data collection, processing and visualization - + Open-source IoT Platform - Device management, data collection, processing and visualization UTF-8 @@ -117,7 +116,7 @@ remote-js-client - org.thingsboard.common + org.thingsboard.common stats @@ -197,6 +196,10 @@ org.apache.commons commons-csv + + org.apache.commons + commons-compress + org.springframework spring-context-support @@ -344,6 +347,11 @@ jdbc test + + org.jeasy + easy-random-core + test + org.javadelight delight-nashorn-sandbox diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 3ded7b7171..dda0f7d758 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -19,16 +19,25 @@ import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -41,6 +50,14 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.tenant.TbTenantService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.tenant.TenantExportService; +import org.thingsboard.server.service.sync.tenant.TenantImportService; +import org.thingsboard.server.service.sync.tenant.util.Result; +import org.thingsboard.server.service.sync.tenant.util.TenantExportConfig; +import org.thingsboard.server.service.sync.tenant.util.TenantImportConfig; + +import java.io.IOException; +import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.HOME_DASHBOARD; import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; @@ -66,6 +83,8 @@ public class TenantController extends BaseController { private final TenantService tenantService; private final TbTenantService tbTenantService; + private final TenantExportService tenantExportService; + private final TenantImportService tenantImportService; @ApiOperation(value = "Get Tenant (getTenantById)", notes = "Fetch the Tenant object based on the provided Tenant Id. " + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @@ -166,4 +185,54 @@ public class TenantController extends BaseController { return checkNotNull(tenantService.findTenantInfos(pageLink)); } + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @PostMapping("/tenant/export") + public UUID exportTenant(@RequestBody TenantExportConfig exportConfig) { + return tenantExportService.exportTenant(exportConfig); + } + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @GetMapping("/tenant/export/result/{tenantId}") + public Result getTenantExportResult(@PathVariable UUID tenantId) { + Result result = tenantExportService.getResult(tenantId); + if (result == null) { + result = new Result<>(); + } + return result; + } + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @DeleteMapping("/tenant/export/{tenantId}") + public void cancelTenantExport(@PathVariable UUID tenantId) { + tenantExportService.cancelExport(tenantId); + } + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @GetMapping("/tenant/export/result/{tenantId}/download") + public ResponseEntity downloadTenantExportData(@PathVariable UUID tenantId) { + return tenantExportService.downloadResult(tenantId); + } + + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @PostMapping(value = "/tenant/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public UUID importTenant(@RequestPart MultipartFile dataFile, + @RequestPart(required = false) TenantImportConfig importConfig) throws IOException { + if (importConfig == null) { + importConfig = new TenantImportConfig(); + } + return tenantImportService.importTenant(dataFile.getInputStream(), importConfig); + } + + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @GetMapping("/tenant/import/result/{tenantId}") + public Result getTenantImportResult(@PathVariable UUID tenantId) { + Result result = tenantImportService.getResult(tenantId); + if (result == null) { + result = new Result<>(); + } + return result; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java new file mode 100644 index 0000000000..62858ddd1b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantExportService.java @@ -0,0 +1,341 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.audit.AuditLog; +import org.thingsboard.server.common.data.event.Event; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKv; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.LatestTsKv; +import org.thingsboard.server.common.data.kv.TsKv; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.attributes.AttributesDao; +import org.thingsboard.server.dao.audit.AuditLogDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.event.EventDao; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.relation.RelationDao; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; +import org.thingsboard.server.service.sync.tenant.util.DataWrapper; +import org.thingsboard.server.service.sync.tenant.util.Result; +import org.thingsboard.server.service.sync.tenant.util.ResultStore; +import org.thingsboard.server.service.sync.tenant.util.Storage; +import org.thingsboard.server.service.sync.tenant.util.TenantExportConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; + +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; +import static org.thingsboard.server.common.data.ObjectType.EVENT; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TS_KV; +import static org.thingsboard.server.common.data.ObjectType.values; + +@Service +@RequiredArgsConstructor +@Slf4j +public class TenantExportService { + + private final Storage storage; + private final EntityDaoRegistry entityDaoRegistry; + private final TenantDao tenantDao; + private final EventDao eventDao; + private final AuditLogDao auditLogDao; + private final AttributesDao attributesDao; + private final RelationDao relationDao; + private final TimeseriesLatestDao timeseriesLatestDao; + private final TimeseriesDao timeseriesDao; + private final SqlPartitioningRepository partitioningRepository; + private final CacheManager cacheManager; + @Value("${cache.specs.tenantExportResults.timeToLiveInMinutes:1440}") + private int resultsTtl; + private ResultStore resultStore; + + private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tenant-export")); + + private Map> customExporters; + private Map> relatedEntitiesExporters; + + private static final Set RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV); + private static final Set IGNORED = EnumSet.of(TENANT); + private static final List EXPORTABLE = Arrays.stream(values()) + .filter(type -> !RELATED.contains(type) && !IGNORED.contains(type)) + .sorted().toList(); + + @PostConstruct + private void init() { + resultStore = ResultStore.builder() + .name("Tenant export") + .ttlInMinutes(resultsTtl) + .persistFrequency(100) + .removalListener((tenantId, result) -> { + if (result.isDone() && result.isSuccess()) { + storage.cleanUpExportData(tenantId); + } + }) + .cacheName(CacheConstants.TENANT_EXPORT_RESULT_CACHE) + .cacheManager(cacheManager) + .build(); + relatedEntitiesExporters = Map.of( + RELATION, this::exportRelations, + EVENT, this::exportEvents, // todo: query by tenant + ATTRIBUTE_KV, this::exportAttributes, + LATEST_TS_KV, this::exportLatestTelemetry + ); + customExporters = Map.of( + TS_KV, this::exportTelemetry, + AUDIT_LOG, this::exportAuditLogs + ); + } + + public UUID exportTenant(TenantExportConfig exportConfig) { + UUID tenantId = exportConfig.getTenantId(); + log.info("[{}] Exporting tenant", tenantId); + Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId); + if (tenant == null) { + throw new IllegalArgumentException("Tenant with id " + tenantId + " not found"); + } + executor.submit(() -> { + try { + exportTenant(tenant, exportConfig); + resultStore.update(tenantId, result -> { + result.setSuccess(true); + result.setDone(true); + }); + } catch (Throwable t) { + log.error("Failed to export tenant {}", tenant, t); + try { + resultStore.update(tenantId, result -> { + result.setError(ExceptionUtils.getStackTrace(t)); + result.getStats().clear(); + result.setDone(true); + }); + storage.cleanUpExportData(tenantId); + } catch (Exception e) { + log.error("Failed to handle export error", e); + } + } + }); + return tenantId; + } + + private void exportTenant(Tenant tenant, TenantExportConfig exportConfig) { + TenantId tenantId = tenant.getId(); + storage.init(tenantId.getId()); + + export(tenantId, TENANT, tenant, exportConfig); + for (ObjectType type : EXPORTABLE) { + if (exportConfig.getSkipped().contains(type)) { + continue; + } + + log.debug("[{}] Exporting {} entities", tenantId, type); + if (!customExporters.containsKey(type)) { + TenantEntityDao dao = entityDaoRegistry.getTenantEntityDao(type); + var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100); + for (Object entity : entities) { + export(tenantId, type, entity, exportConfig); + } + } else { + customExporters.get(type).accept(tenantId, exportConfig); + } + resultStore.flush(tenantId.getId(), type); + } + resultStore.flush(tenantId.getId(), RELATED.toArray(ObjectType[]::new)); + + storage.archiveExportData(tenantId.getId()); + } + + private void export(TenantId tenantId, ObjectType type, Object entity, TenantExportConfig exportConfig) { + save(tenantId, type, entity); + if (entity instanceof HasId hasId && hasId.getId() instanceof EntityId entityId) { + relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> { + if (!exportConfig.getSkipped().contains(relatedEntityType)) { + exporter.accept(tenantId, entityId); + } + }); + } + } + + private void save(TenantId tenantId, ObjectType type, Object entity) { + DataWrapper dataWrapper = DataWrapper.of(entity); + + storage.save(tenantId.getId(), type, dataWrapper); + resultStore.report(tenantId.getId(), type); + log.trace("[{}][{}] Saved entity {}", tenantId, type, entity); + } + + private Map getPartitions(String table) { + List partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList(); + if (partitionsStartTime.isEmpty()) { + return Collections.emptyMap(); + } + + Map partitions = new HashMap<>(); + for (int i = 0; i < partitionsStartTime.size(); i++) { + Long startTime = partitionsStartTime.get(i); + Long endTime; + if (partitionsStartTime.size() - 1 == i) { + endTime = System.currentTimeMillis(); + } else { + endTime = partitionsStartTime.get(i + 1) - 1; + } + partitions.put(startTime, endTime); + } + return partitions; + } + + public Result getResult(UUID tenantId) { + return resultStore.getStoredResult(tenantId); + } + + public void cancelExport(UUID tenantId) { + boolean removed = storage.cleanUpExportData(tenantId); + if (!removed) { + throw new IllegalArgumentException("Not found"); + } + } + + @SneakyThrows + public ResponseEntity downloadResult(UUID tenantId) { + var result = resultStore.getStoredResult(tenantId); + if (result == null || !result.isDone()) { + throw new IllegalStateException("Not ready yet"); + } else if (!result.isSuccess()) { + throw new IllegalStateException("Tenant export failed: " + result.getError()); + } + + String fileName = "tenant_export_data_" + tenantId + ".tar"; + return ResponseEntity.ok() + .header("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + fileName) + .header("x-filename", fileName) + .body(new InputStreamResource(storage.downloadExportData(tenantId))); + } + + + private void exportTelemetry(TenantId tenantId, TenantExportConfig exportConfig) { + storage.readAndProcess(LATEST_TS_KV, tenantId.getId(), dataWrapper -> { + LatestTsKv latestTsKv = (LatestTsKv) dataWrapper.getEntity(); + EntityId entityId = latestTsKv.getEntityId(); + String key = latestTsKv.getEntry().getKey(); + + // todo: submit for 10 latest kv, wait + try { + timeseriesDao.findAllAsync(tenantId, entityId, key, tsKvEntry -> { + TsKv tsKv = new TsKv(entityId, tsKvEntry, exportConfig.getTsKvTtl()); + save(tenantId, TS_KV, tsKv); + }).get(60, TimeUnit.MINUTES); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private void exportAuditLogs(TenantId tenantId, TenantExportConfig exportConfig) { + Map partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME); + partitions.forEach((startTime, endTime) -> { + PageDataIterable auditLogs = new PageDataIterable<>(pageLink -> { + return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (AuditLog auditLog : auditLogs) { + save(tenantId, AUDIT_LOG, auditLog); + } + }); + } + + private void exportAttributes(TenantId tenantId, EntityId entityId) { + for (AttributeScope attributeScope : AttributeScope.values()) { + List attributes = attributesDao.findAll(tenantId, entityId, attributeScope); + for (AttributeKvEntry entry : attributes) { + AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry); + save(tenantId, ATTRIBUTE_KV, attributeKv); + } + } + } + + private void exportRelations(TenantId tenantId, EntityId entityId) { + List relations = relationDao.findAllByFrom(tenantId, entityId); + for (EntityRelation relation : relations) { + save(tenantId, RELATION, relation); + } + } + + @SneakyThrows + private void exportLatestTelemetry(TenantId tenantId, EntityId entityId) { + List latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS); + for (TsKvEntry tsKvEntry : latestTelemetry) { + LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry); + save(tenantId, LATEST_TS_KV, latestTsKv); + } + } + + private void exportEvents(TenantId tenantId, EntityId entityId) { + for (EventType eventType : EventType.values()) { + Map partitions = getPartitions(eventType.getTable()); + partitions.forEach((startTime, endTime) -> { + PageDataIterable events = new PageDataIterable<>(pageLink -> { + return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime)); + }, 512); + for (Event event : events) { + save(tenantId, EVENT, event); + } + }); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantImportService.java new file mode 100644 index 0000000000..61d50f12ef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/TenantImportService.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant; + +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.tenant.TenantProfileService; +import org.thingsboard.server.service.sync.tenant.util.DataWrapper; +import org.thingsboard.server.service.sync.tenant.util.Result; +import org.thingsboard.server.service.sync.tenant.util.ResultStore; +import org.thingsboard.server.service.sync.tenant.util.Storage; +import org.thingsboard.server.service.sync.tenant.util.TenantImportConfig; + +import java.io.InputStream; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; + +import static org.thingsboard.server.common.data.ObjectType.TENANT; + +@Service +@RequiredArgsConstructor +@Slf4j +public class TenantImportService { + + private final Storage storage; + private final EntityDaoRegistry entityDaoRegistry; + private final TenantProfileService tenantProfileService; + private final TransactionTemplate transactionTemplate; + private final CacheManager cacheManager; + private ResultStore resultStore; + @Value("${cache.specs.tenantImportResults.timeToLiveInMinutes:1440}") + private int resultsTtl; + + private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tenant-import")); + + @PostConstruct + private void init() { + resultStore = ResultStore.builder() + .name("Tenant import") + .ttlInMinutes(resultsTtl) + .persistFrequency(100) + .cacheName(CacheConstants.TENANT_IMPORT_RESULT_CACHE) + .cacheManager(cacheManager) + .build(); + } + + public UUID importTenant(InputStream dataStream, TenantImportConfig config) { + storage.unwrapImportData(dataStream); + + AtomicReference tenant = new AtomicReference<>(); + storage.readAndProcess(TENANT, dataWrapper -> { + tenant.set((Tenant) dataWrapper.getEntity()); + }); + if (tenant.get() == null) { + throw new IllegalStateException("No tenant data found"); + } + UUID tenantId = tenant.get().getUuidId(); + + executor.submit(() -> { + try { + transactionTemplate.executeWithoutResult(status -> { + importTenant(tenant.get(), config); + }); + resultStore.update(tenantId, result -> { + result.setSuccess(true); + result.setDone(true); + }); + } catch (Throwable t) { + log.error("Failed to import tenant", t); + try { + resultStore.update(tenantId, result -> { + result.setError(ExceptionUtils.getStackTrace(t)); + result.setDone(true); + }); + } catch (Exception e) { + log.error("Failed to handle import error", e); + } + } + storage.cleanUpImportData(); + }); + + return tenantId; + } + + private void importTenant(Tenant tenant, TenantImportConfig config) { + TenantId tenantId = tenant.getId(); + for (ObjectType type : ObjectType.values()) { // TODO: in parallel for related entities + ts kv + log.debug("[{}] Importing {} entities", tenantId, type); + storage.readAndProcess(type, dataWrapper -> { + save(tenantId, type, dataWrapper); + }); + resultStore.flush(tenantId.getId(), type); + } + + clearCaches(); + } + + private void save(TenantId tenantId, ObjectType type, DataWrapper dataWrapper) { + Object entity = dataWrapper.getEntity(); + if (entity instanceof Tenant tenant) { + tenant.setTenantProfileId(tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID).getId()); + } + + entityDaoRegistry.getDao(type).save(tenantId, entity); + resultStore.report(tenantId.getId(), type); + log.trace("[{}][{}] Imported entity {}", tenantId, type, entity); + } + + private void clearCaches() { + cacheManager.getCacheNames().forEach(cacheName -> cacheManager.getCache(cacheName).clear()); + } + + public Result getResult(UUID tenantId) { + return resultStore.getStoredResult(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/DataWrapper.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/DataWrapper.java new file mode 100644 index 0000000000..b881d08d34 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/DataWrapper.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.Map; + +@Data +@NoArgsConstructor +public class DataWrapper { + + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + private Object entity; + + private Map additionalInfo; + + private DataWrapper(Object entity) { + this.entity = entity; + } + + public static DataWrapper of(Object entity) { + return new DataWrapper(entity); + } + + public void putAdditionalInfo(String key, String value) { + if (additionalInfo == null) { + additionalInfo = new HashMap<>(); + } + additionalInfo.put(key, value); + } + + public String getAdditionalInfo(String key) { + if (additionalInfo == null) { + return null; + } + return additionalInfo.get(key); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Result.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Result.java new file mode 100644 index 0000000000..8cfb45c38f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Result.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import lombok.Data; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +public class Result implements Serializable { + + private boolean done; + private boolean success; + private String error; + + private final Map stats = new LinkedHashMap<>(); + + public int report(K key) { + return stats.computeIfAbsent(key, k -> new AtomicInteger()).incrementAndGet(); + } + + public int getCount(K key) { + return Optional.ofNullable(stats.get(key)).map(AtomicInteger::get).orElse(0); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/ResultStore.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/ResultStore.java new file mode 100644 index 0000000000..dde7034dd4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/ResultStore.java @@ -0,0 +1,103 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.CacheManager; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +@Slf4j +public class ResultStore { + + private final String name; + private final int persistFrequency; + + private final Cache> results; + private final org.springframework.cache.Cache resultsCache; + + @Builder + public ResultStore(String name, int ttlInMinutes, int persistFrequency, BiConsumer> removalListener, + String cacheName, CacheManager cacheManager) { + this.name = name; + this.persistFrequency = persistFrequency; + + this.resultsCache = Optional.ofNullable(cacheManager.getCache(cacheName)) + .orElseThrow(() -> new IllegalArgumentException(cacheName + " cache is missing")); + this.results = Caffeine.newBuilder() + .expireAfterAccess(ttlInMinutes, TimeUnit.MINUTES) + .>removalListener((key, result, removalCause) -> { + if (key != null) { + if (removalListener != null) { + removalListener.accept(key, result); + } + resultsCache.evict(key); + } + }) + .build(); + } + + public void report(UUID id, K key) { + Result result = getOrCreateResult(id); + int count = result.report(key); + if (count % persistFrequency == 0) { + resultsCache.put(id, result); + print(id, key, count); + } + } + + public int get(UUID id, K key) { + Result result = getOrCreateResult(id); + return result.getCount(key); + } + + public void update(UUID id, Consumer> updater) { + Result result = getOrCreateResult(id); + updater.accept(result); + flush(id); + } + + public void flush(UUID id, K... keys) { + Result result = getOrCreateResult(id); + resultsCache.put(id, result); + for (K key : keys) { + int count = result.getCount(key); + if (count % persistFrequency != 0) { // if we didn't print it before + print(id, key, count); + } + } + } + + private void print(UUID id, K key, int count) { + log.info("[{}][{}][{}] {} processed", name, id, key, count); + } + + private Result getOrCreateResult(UUID id) { + return results.get(id, k -> new Result()); + } + + public Result getStoredResult(UUID id) { + return resultsCache.get(id, Result.class); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Storage.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Storage.java new file mode 100644 index 0000000000..e47caf9b3e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/Storage.java @@ -0,0 +1,193 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; + +import java.io.BufferedReader; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.UncheckedIOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +@Service +@Slf4j +public class Storage { + + @Value("${tenant.export.working_directory:/tmp/tenants-export-import}") + private String workingDirectory; + + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + private final Map> files = new ConcurrentHashMap<>(); + + @SneakyThrows + public void init(UUID tenantId) { + Path workingDirectory = Path.of(this.workingDirectory, tenantId.toString()); + Files.createDirectories(workingDirectory); + files.put(tenantId, new HashMap<>()); + } + + @SneakyThrows + public void save(UUID tenantId, ObjectType type, DataWrapper dataWrapper) { + Path file = getExportDataPath(tenantId, type); + Map dataFiles = files.get(tenantId); + if (dataFiles == null) { + throw new IllegalStateException("Export cancelled"); + } + Writer writer = dataFiles.computeIfAbsent(file, path -> { + try { + Files.deleteIfExists(file); + Files.createFile(file); + + FileOutputStream fileOutputStream = new FileOutputStream(file.toFile()); + return new OutputStreamWriter(new GZIPOutputStream(fileOutputStream), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }); + + String data = toJson(dataWrapper); + writer.write(data + System.lineSeparator()); + log.trace("[{}] Saved entity to {}: {}", tenantId, file, dataWrapper.getEntity()); + } + + @SneakyThrows + public void archiveExportData(UUID tenantId) { + for (Writer writer : files.get(tenantId).values()) { + writer.close(); + } + + Map files = this.files.remove(tenantId); + TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(new FileOutputStream(Path.of(workingDirectory, tenantId.toString(), "data.tar").toFile())); + for (Path file : files.keySet()) { + TarArchiveEntry archiveEntry = new TarArchiveEntry(file, file.getFileName().toString()); + tarArchive.putArchiveEntry(archiveEntry); + Files.copy(file, tarArchive); + tarArchive.closeArchiveEntry(); + Files.delete(file); + } + tarArchive.close(); + } + + @SneakyThrows + public boolean cleanUpExportData(UUID tenantId) { + Path workingDirectory = Path.of(this.workingDirectory, tenantId.toString()); + deleteDirectory(workingDirectory); + return files.remove(tenantId) != null; + } + + @SneakyThrows + public InputStream downloadExportData(UUID tenantId) { + return Files.newInputStream(Path.of(workingDirectory, tenantId.toString(), "data.tar")); + } + + @SneakyThrows + public void unwrapImportData(InputStream dataStream) { + Path workingDirectory = Path.of(this.workingDirectory, "importing"); + deleteDirectory(workingDirectory); + Files.createDirectories(workingDirectory); + + TarArchiveInputStream tarArchive = new TarArchiveInputStream(dataStream); + TarArchiveEntry archiveEntry; + while ((archiveEntry = tarArchive.getNextEntry()) != null) { + Path file = workingDirectory.resolve(archiveEntry.getName()); + Files.createFile(file); + IOUtils.copy(tarArchive, Files.newOutputStream(file)); + } + } + + @SneakyThrows + public void cleanUpImportData() { + Path workingDirectory = Path.of(this.workingDirectory, "importing"); + deleteDirectory(workingDirectory); + } + + @SneakyThrows + public void readAndProcess(ObjectType type, UUID tenantId, Consumer processor) { + Path file = getExportDataPath(tenantId, type); + Writer writer = files.get(tenantId).get(file); + if (writer != null) { + writer.close(); + } + readAndProcess(type, file, processor); + } + + public void readAndProcess(ObjectType type, Consumer processor) { + Path file = Path.of(this.workingDirectory, "importing", type.name().toLowerCase() + ".gz"); + readAndProcess(type, file, processor); + } + + @SneakyThrows + private void readAndProcess(ObjectType type, Path file, Consumer processor) { + if (!Files.exists(file)) { + log.debug("No data for {}", type); + return; + } + + FileInputStream fileInputStream = new FileInputStream(file.toFile()); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(fileInputStream)))) { + reader.lines().forEach(line -> { + DataWrapper dataWrapper; + try { + dataWrapper = jsonMapper.readValue(line, DataWrapper.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + processor.accept(dataWrapper); + }); + } + } + + @SneakyThrows + private String toJson(Object o) { + return jsonMapper.writeValueAsString(o); + } + + private Path getExportDataPath(UUID tenantId, ObjectType type) { + return Path.of(workingDirectory, tenantId.toString(), type.name().toLowerCase() + ".gz"); + } + + private void deleteDirectory(Path directory) throws IOException { + FileUtils.deleteDirectory(directory.toFile()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantExportConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantExportConfig.java new file mode 100644 index 0000000000..dfb3f0cc7d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantExportConfig.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import lombok.Data; +import org.thingsboard.server.common.data.ObjectType; + +import java.util.Collections; +import java.util.Set; +import java.util.UUID; + +@Data +public class TenantExportConfig { + + private UUID tenantId; + private Set skipped; + + private long tsKvTtl; + + public Set getSkipped() { + return skipped != null ? skipped : Collections.emptySet(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantImportConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantImportConfig.java new file mode 100644 index 0000000000..0064b0e816 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/tenant/util/TenantImportConfig.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.tenant.util; + +import lombok.Data; + +@Data +public class TenantImportConfig { + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index c3d28f9fce..624857150c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1772,7 +1772,7 @@ queue: api: # Whether to forward entity data query requests to EDQS (otherwise use PostgreSQL implementation) supported: "${TB_EDQS_API_SUPPORTED:false}" - # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished + # Whether to auto-enable EDQS API (if queue.edqs.api.supported is true) when sync of data to Kafka is finished auto_enable: "${TB_EDQS_API_AUTO_ENABLE:true}" # Interval in milliseconds to check for ready EDQS servers readiness_check_interval: "${TB_EDQS_READINESS_CHECK_INTERVAL_MS:60000}" diff --git a/application/src/test/java/org/thingsboard/server/service/tenant/TenantExportImportTest.java b/application/src/test/java/org/thingsboard/server/service/tenant/TenantExportImportTest.java new file mode 100644 index 0000000000..ee4c9fa125 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/tenant/TenantExportImportTest.java @@ -0,0 +1,383 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.tenant; + +import com.fasterxml.jackson.core.type.TypeReference; +import lombok.SneakyThrows; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpMethod; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.audit.AuditLog; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.audit.AuditLogDao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.housekeeper.CleanUpService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.service.sync.tenant.util.Result; +import org.thingsboard.server.service.sync.tenant.util.TenantExportConfig; +import org.thingsboard.server.utils.TestUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.ObjectType.ADMIN_SETTINGS; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; +import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; +import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; +import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; +import static org.thingsboard.server.common.data.ObjectType.EVENT; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.OTA_PACKAGE; +import static org.thingsboard.server.common.data.ObjectType.QUEUE; +import static org.thingsboard.server.common.data.ObjectType.RPC; +import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; +import static org.thingsboard.server.common.data.ObjectType.RULE_NODE; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TS_KV; +import static org.thingsboard.server.utils.TestUtils.newRandomizedEntity; + +@DaoSqlTest +@SuppressWarnings("unchecked") +public class TenantExportImportTest extends AbstractControllerTest { + + @Autowired + private EventService eventService; + @Autowired + private AuditLogDao auditLogDao; + @Autowired + private TimeseriesService timeseriesService; + @Autowired + private AttributesService attributesService; + @Autowired + private RelationService relationService; + @Autowired + private EntityDaoRegistry entityDaoRegistry; + @Autowired + private TenantDao tenantDao; + @Autowired + private CleanUpService cleanUpService; + @Autowired + private JdbcTemplate jdbcTemplate; + + private TenantId tenantId; + private Map> createdEntities; + private Map> createdRelatedEntities; + + @Before + public void before() throws Exception { + createdEntities = new HashMap<>(); + createdRelatedEntities = new HashMap<>(); + loginSysAdmin(); + } + + @After + public void after() { + + } + + @Test + public void testExportImport() throws Exception { + Tenant tenant = newRandomizedEntity(Tenant.class); + tenant.setTenantProfileId(tenantProfileId); + this.tenantId = tenant.getId(); + saveEntity(tenant, TENANT); + fillTenant(); + + Result exportResult = exportTenant(tenantId); + createdEntities.forEach((type, entities) -> { + int exportedCount = exportResult.getCount(type); + if (exportedCount != entities.size()) { + fail("expected " + entities.size() + " exported entities of type " + type + " but was " + exportedCount + ". " + + "expected: " + entities + ", actual: " + findAllOfType(type)); + return; + } + assertAllPresent(type, entities); + }); + byte[] exportData = doGet("/api/tenant/export/result/" + tenantId + "/download").andExpect(status().isOk()).andReturn().getResponse().getContentAsByteArray(); + + loginSysAdmin(); + createdEntities.values().forEach(entities -> { + for (HasId entity : entities) { + if (entity.getId() instanceof EntityId entityId) { + cleanUpService.cleanUpRelatedData(tenantId, entityId); + } + } + }); + jdbcTemplate.execute("TRUNCATE TABLE audit_log"); + deleteTenant(tenantId); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + createdEntities.forEach((type, entities) -> { + assertThat(findAllOfType(type)).isEmpty(); + entities.forEach(entity -> { + if (entity.getId() instanceof EntityId entityId) { + assertThat(findAuditLogs(entityId)).isEmpty(); + assertThat(findEvents(entityId)).isEmpty(); + assertThat(findAttributes(entityId)).isEmpty(); + assertThat(findLatestTs(entityId)).isEmpty(); + TsKvEntry latestKv = (TsKvEntry) createdRelatedEntities.get(LATEST_TS_KV).get(entityId); + assertThat(findTsHistory(entityId, latestKv.getKey())).isEmpty(); + } + }); + }); + }); + + Result importResult = importTenant(exportData); + createdEntities.forEach((type, entities) -> { + int importedCount = importResult.getCount(type); + if (importedCount != entities.size()) { + fail("expected " + entities.size() + " imported entities of type " + type + " but was " + importedCount + ". " + + "expected: " + entities + ", actual: " + findAllOfType(type)); + return; + } + assertAllPresent(type, entities); + }); + } + + private void assertAllPresent(ObjectType type, List expectedEntities) { + assertThat(findAllOfType(type)).containsAll(expectedEntities); + expectedEntities.forEach(entity -> { + if (entity.getId() instanceof EntityId entityId) { + assertThat((List) findAttributes(entityId)).containsExactly(createdRelatedEntities.get(ATTRIBUTE_KV).get(entityId)); + List latestTs = findLatestTs(entityId); + assertThat((List) latestTs).containsExactly(createdRelatedEntities.get(LATEST_TS_KV).get(entityId)); + assertThat((List) findTsHistory(entityId, latestTs.get(0).getKey())).containsExactly(createdRelatedEntities.get(TS_KV).get(entityId)); + assertThat((List) findAuditLogs(entityId)).containsExactly(createdRelatedEntities.get(AUDIT_LOG).get(entityId)); + assertThat((List) findEvents(entityId)).containsExactly(createdRelatedEntities.get(EVENT).get(entityId)); + } + }); + } + + private Result exportTenant(TenantId tenantId) { + TenantExportConfig exportConfig = new TenantExportConfig(); + exportConfig.setTenantId(tenantId.getId()); + doPost("/api/tenant/export", exportConfig, UUID.class); + Result result = await().atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> doGetTyped("/api/tenant/export/result/" + tenantId.getId(), new TypeReference<>() {}), Result::isDone); + System.err.println(result); + if (!result.isSuccess()) { + throw new RuntimeException(result.getError()); + } + return result; + } + + private Result importTenant(byte[] data) throws Exception { + MockMultipartFile dataFile = new MockMultipartFile("dataFile", "data.tar", "application/octet-stream", data); + var request = MockMvcRequestBuilders.multipart(HttpMethod.POST, "/api/tenant/import").file(dataFile); + setJwtToken(request); + UUID tenantId = readResponse(mockMvc.perform(request).andExpect(status().isOk()), UUID.class); + + Result result = await().atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> doGetTyped("/api/tenant/import/result/" + tenantId, new TypeReference<>() {}), Result::isDone); + System.err.println(result); + if (!result.isSuccess()) { + throw new RuntimeException(result.getError()); + } + return result; + } + + private void fillTenant() { + Customer customer = newRandomizedEntity(Customer.class); + customer.setTenantId(tenantId); + saveEntity(customer, CUSTOMER); + + AdminSettings adminSettings = newRandomizedEntity(AdminSettings.class); + adminSettings.setTenantId(tenantId); + saveEntity(adminSettings, ADMIN_SETTINGS); + + Queue queue = newRandomizedEntity(Queue.class); + queue.setTenantId(tenantId); + saveEntity(queue, QUEUE); + + Rpc rpc = newRandomizedEntity(Rpc.class); + rpc.setTenantId(tenantId); + saveEntity(rpc, RPC); + + RuleChain ruleChain = newRandomizedEntity(RuleChain.class); + ruleChain.setTenantId(tenantId); + saveEntity(ruleChain, RULE_CHAIN); + + for (int i = 0; i < 10; i++) { + RuleNode ruleNode = newRandomizedEntity(RuleNode.class); + ruleNode.setRuleChainId(ruleChain.getId()); + saveEntity(ruleNode, RULE_NODE); + createRelation(ruleNode.getId(), ruleChain.getId()); + createRelation(ruleChain.getId(), ruleNode.getId()); + } + + Dashboard dashboard = newRandomizedEntity(Dashboard.class); + dashboard.setTenantId(tenantId); + saveEntity(dashboard, DASHBOARD); + + DeviceProfile deviceProfile = newRandomizedEntity(DeviceProfile.class); + + OtaPackage otaPackage = newRandomizedEntity(OtaPackage.class); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfile.getId()); + saveEntity(otaPackage, OTA_PACKAGE); + + deviceProfile.setTenantId(tenantId); + deviceProfile.setFirmwareId(otaPackage.getId()); + deviceProfile.setSoftwareId(otaPackage.getId()); + deviceProfile.setDefaultEdgeRuleChainId(null); + deviceProfile.setDefaultRuleChainId(ruleChain.getId()); + deviceProfile.setDefaultDashboardId(dashboard.getId()); + saveEntity(deviceProfile, DEVICE_PROFILE); + } + + private > T saveEntity(T entity, ObjectType type) { + entity = (T) entityDaoRegistry.getDao(type).save(tenantId, entity); + if (entity.getId() instanceof EntityId entityId) { + createAttribute(entityId); + createTelemetry(entityId); + createAuditLog(entityId); + createEvent(entityId); + } + createdEntities.computeIfAbsent(type, k -> new ArrayList<>()).add(entity); + return entity; + } + + private List findAllOfType(ObjectType type) { + if (type == TENANT) { + Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); + if (tenant == null) { + return Collections.emptyList(); + } + return List.of(tenant); + } + TenantEntityDao dao = entityDaoRegistry.getTenantEntityDao(type); + PageData stored = dao.findAllByTenantId(tenantId, new PageLink(1000)); + return stored.getData(); + } + + private void createRelation(EntityId to, EntityId from) { + EntityRelation relation = TestUtils.newRandomizedEntity(EntityRelation.class); + relation.setTo(to); + relation.setFrom(from); + relationService.saveRelation(tenantId, relation); + } + + @SneakyThrows + private void createAttribute(EntityId entityId) { + BaseAttributeKvEntry attributeKv = newRandomizedEntity(BaseAttributeKvEntry.class); + attributesService.save(tenantId, entityId, AttributeScope.SHARED_SCOPE, attributeKv).get(); + createdRelatedEntities.computeIfAbsent(ATTRIBUTE_KV, k -> new HashMap<>()).put(entityId, attributeKv); + } + + @SneakyThrows + private List findAttributes(EntityId entityId) { + return attributesService.findAll(tenantId, entityId, AttributeScope.SHARED_SCOPE).get(); + } + + @SneakyThrows + private void createTelemetry(EntityId entityId) { + BasicTsKvEntry tsKvEntry = newRandomizedEntity(BasicTsKvEntry.class); + timeseriesService.save(tenantId, entityId, tsKvEntry).get(); + createdRelatedEntities.computeIfAbsent(TS_KV, k -> new HashMap<>()).put(entityId, tsKvEntry); + createdRelatedEntities.computeIfAbsent(LATEST_TS_KV, k -> new HashMap<>()).put(entityId, tsKvEntry); + } + + @SneakyThrows + private List findLatestTs(EntityId entityId) { + return timeseriesService.findAllLatest(tenantId, entityId).get(); + } + + @SneakyThrows + private List findTsHistory(EntityId entityId, String key) { + return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(key, 0, System.currentTimeMillis(), 100, "ASC"))).get(); + } + + @SneakyThrows + private void createEvent(EntityId entityId) { + LifecycleEvent event = LifecycleEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId("test") + .lcEventType("test") + .success(true) + .build(); + eventService.saveAsync(event).get(); + createdRelatedEntities.computeIfAbsent(EVENT, k -> new HashMap<>()).put(entityId, findEvents(entityId).get(0)); + } + + private List findEvents(EntityId entityId) { + return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100, 0)).getData(); + } + + @SneakyThrows + private void createAuditLog(EntityId entityId) { + AuditLog auditLog = newRandomizedEntity(AuditLog.class); + auditLog.setTenantId(tenantId); + auditLog.setEntityId(entityId); + auditLog = auditLogDao.save(tenantId, auditLog); + createdRelatedEntities.computeIfAbsent(AUDIT_LOG, k -> new HashMap<>()).put(entityId, auditLog); + } + + private List findAuditLogs(EntityId entityId) { + return auditLogDao.findAuditLogsByTenantIdAndEntityId(tenantId.getId(), entityId, Arrays.stream(ActionType.values()).toList(), new TimePageLink(100, 0)).getData(); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/utils/TestUtils.java b/application/src/test/java/org/thingsboard/server/utils/TestUtils.java new file mode 100644 index 0000000000..15aaf19027 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/utils/TestUtils.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.jeasy.random.EasyRandom; +import org.jeasy.random.EasyRandomParameters; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.AllowCreateNewDevicesDeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; +import org.thingsboard.server.common.data.device.profile.DeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileProvisionConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; + +import java.nio.ByteBuffer; +import java.util.UUID; + +public class TestUtils { + + static EasyRandom easyRandom; + + static { + EasyRandomParameters parameters = new EasyRandomParameters() + .randomize(DeviceConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceConfiguration.class)) + .randomize(DeviceTransportConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceTransportConfiguration.class)) + .randomize(DeviceProfileConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceProfileConfiguration.class)) + .randomize(DeviceProfileTransportConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceProfileTransportConfiguration.class)) + .randomize(DeviceProfileProvisionConfiguration.class, () -> easyRandom.nextObject(AllowCreateNewDevicesDeviceProfileProvisionConfiguration.class)) + .randomize(DeviceProfileAlarm.class, DeviceProfileAlarm::new) + .randomize(JsonNode.class, () -> JacksonUtil.newObjectNode() + .put(RandomStringUtils.randomAlphanumeric(10), RandomStringUtils.randomAlphanumeric(10)) + .put(RandomStringUtils.randomAlphanumeric(10), easyRandom.nextDouble())) + .randomize(EntityId.class, () -> new DeviceId(UUID.randomUUID())) + .randomize(KvEntry.class, () -> easyRandom.nextBoolean() ? easyRandom.nextObject(StringDataEntry.class) : easyRandom.nextObject(DoubleDataEntry.class)) + .randomize(Long.class, () -> RandomUtils.nextLong(1, Integer.MAX_VALUE)) + .randomize(ByteBuffer.class, () -> { + byte[] bytes = new byte[50]; + easyRandom.nextBytes(bytes); + return ByteBuffer.wrap(bytes); + }); + + easyRandom = new EasyRandom(parameters); + } + + public static T newRandomizedEntity(Class type) { + return easyRandom.nextObject(type); + } + +} diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 13c93da411..218e4e5127 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -13,7 +13,7 @@ - + @@ -29,9 +29,9 @@ - - - + + + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 12c8e5053b..4d84c7abed 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -51,5 +51,8 @@ public interface OtaPackageService extends EntityDaoService { void deleteOtaPackagesByTenantId(TenantId tenantId); + void deleteOtaPackagesByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId); + long sumDataSizeByTenantId(TenantId tenantId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 5b167c88a2..957f06c2b5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -54,4 +54,7 @@ public class CacheConstants { public static final String ALARM_TYPES_CACHE = "alarmTypes"; public static final String QR_CODE_SETTINGS_CACHE = "qrCodeSettings"; public static final String MOBILE_SECRET_KEY_CACHE = "mobileSecretKey"; + public static final String TENANT_EXPORT_RESULT_CACHE = "tenantExportResults"; + public static final String TENANT_IMPORT_RESULT_CACHE = "tenantImportResults"; + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index 51c631fe4f..49671e7a9d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -23,17 +23,25 @@ public enum ObjectType { TENANT, TENANT_PROFILE, CUSTOMER, + ADMIN_SETTINGS, QUEUE, RPC, RULE_CHAIN, + RULE_NODE, + RULE_NODE_STATE, OTA_PACKAGE, RESOURCE, EVENT, RULE_NODE, USER, + USER_CREDENTIALS, + USER_SETTINGS, + USER_AUTH_SETTINGS, EDGE, + EDGE_EVENT, WIDGETS_BUNDLE, WIDGET_TYPE, + WIDGETS_BUNDLE_WIDGET, DASHBOARD, DEVICE_PROFILE, DEVICE, @@ -53,11 +61,14 @@ public enum ObjectType { ALARM_COMMENT, API_USAGE_STATE, QUEUE_STATS, - + ALARM_TYPE, + MOBILE_APP_SETTINGS, + EVENT, AUDIT_LOG, RELATION, ATTRIBUTE_KV, - LATEST_TS_KV; + LATEST_TS_KV, + TS_KV; public static final Set edqsTenantTypes = EnumSet.of( TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java index 26376bbda3..7577dd0815 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java @@ -15,12 +15,16 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonSetter; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.OtaPackageId; import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.Optional; @Schema @Data @@ -44,4 +48,21 @@ public class OtaPackage extends OtaPackageInfo { super(otaPackage); this.data = otaPackage.getData(); } + + @JsonGetter("data") + public String getEncodedData() { + return Optional.ofNullable(data) + .map(ByteBuffer::array) + .map(Base64.getEncoder()::encodeToString) + .orElse(null); + } + + @JsonSetter("data") + public void setEncodedData(String data) { + this.data = Optional.ofNullable(data) + .map(Base64.getDecoder()::decode) + .map(ByteBuffer::wrap) + .orElse(null); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java new file mode 100644 index 0000000000..0813e72123 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/alarm/AlarmType.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.alarm; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class AlarmType { + + private TenantId tenantId; + private String type; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java index 452c1d6574..7b73a4d45f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/ErrorEvent.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -23,6 +25,7 @@ import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -40,6 +43,15 @@ public class ErrorEvent extends Event { this.error = error; } + @JsonCreator + private ErrorEvent(@JsonProperty("tenantId") TenantId tenantId, @JsonProperty("entityId") UUID entityId, + @JsonProperty("serviceId") String serviceId, @JsonProperty("id") EventId id, @JsonProperty("ts") long ts, + @JsonProperty("method") String method, @JsonProperty("error") String error) { + super(tenantId, entityId, serviceId, id, ts); + this.method = method; + this.error = error; + } + @Getter @Setter private String method; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java index 61e3d052bd..62c8f0dd4b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/Event.java @@ -49,6 +49,14 @@ public abstract class Event extends BaseData { this.createdTime = ts; } + public Event(TenantId tenantId, UUID entityId, String serviceId, EventId id, long ts) { + super(id); + this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID; + this.entityId = entityId; + this.serviceId = serviceId; + this.createdTime = ts; + } + public abstract EventType getType(); public EventInfo toInfo(EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java index e97fb17677..bc114031b0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/LifecycleEvent.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -23,6 +25,7 @@ import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -43,6 +46,16 @@ public class LifecycleEvent extends Event { this.error = error; } + @JsonCreator + private LifecycleEvent(@JsonProperty("tenantId") TenantId tenantId, @JsonProperty("entityId") UUID entityId, + @JsonProperty("serviceId") String serviceId, @JsonProperty("id") EventId id, @JsonProperty("ts") long ts, + @JsonProperty("lcEventType") String lcEventType, @JsonProperty("success") boolean success, @JsonProperty("error") String error) { + super(tenantId, entityId, serviceId, id, ts); + this.lcEventType = lcEventType; + this.success = success; + this.error = error; + } + @Getter private final String lcEventType; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java index 5872e05564..10ab2cecc3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleChainDebugEvent.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -23,6 +25,7 @@ import lombok.Setter; import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -40,6 +43,15 @@ public class RuleChainDebugEvent extends Event { this.error = error; } + @JsonCreator + private RuleChainDebugEvent(@JsonProperty("tenantId") TenantId tenantId, @JsonProperty("entityId") UUID entityId, + @JsonProperty("serviceId") String serviceId, @JsonProperty("id") EventId id, @JsonProperty("ts") long ts, + @JsonProperty("message") String message, @JsonProperty("error") String error) { + super(tenantId, entityId, serviceId, id, ts); + this.message = message; + this.error = error; + } + @Getter @Setter private String message; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java index 5dd4f2cab8..f0c0eafbfa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/RuleNodeDebugEvent.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -24,6 +26,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -51,6 +54,24 @@ public class RuleNodeDebugEvent extends Event { this.error = error; } + @JsonCreator + private RuleNodeDebugEvent(@JsonProperty("tenantId") TenantId tenantId, @JsonProperty("entityId") UUID entityId, + @JsonProperty("serviceId") String serviceId, @JsonProperty("id") EventId id, @JsonProperty("ts") long ts, + @JsonProperty("eventType") String eventType, @JsonProperty("eventEntity") EntityId eventEntity, @JsonProperty("msgId") UUID msgId, + @JsonProperty("msgType") String msgType, @JsonProperty("dataType") String dataType, @JsonProperty("relationType") String relationType, + @JsonProperty("data") String data, @JsonProperty("metadata") String metadata, @JsonProperty("error") String error) { + super(tenantId, entityId, serviceId, id, ts); + this.eventType = eventType; + this.eventEntity = eventEntity; + this.msgId = msgId; + this.msgType = msgType; + this.dataType = dataType; + this.relationType = relationType; + this.data = data; + this.metadata = metadata; + this.error = error; + } + @Getter private final String eventType; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java index e71b44185f..547c8554e8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/event/StatisticsEvent.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.event; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Builder; import lombok.EqualsAndHashCode; @@ -22,6 +24,7 @@ import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.id.EventId; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; @@ -39,6 +42,15 @@ public class StatisticsEvent extends Event { this.errorsOccurred = errorsOccurred; } + @JsonCreator + private StatisticsEvent(@JsonProperty("tenantId") TenantId tenantId, @JsonProperty("entityId") UUID entityId, + @JsonProperty("serviceId") String serviceId, @JsonProperty("id") EventId id, @JsonProperty("ts") long ts, + @JsonProperty("messagesProcessed") long messagesProcessed, @JsonProperty("errorsOccurred") long errorsOccurred) { + super(tenantId, entityId, serviceId, id, ts); + this.messagesProcessed = messagesProcessed; + this.errorsOccurred = errorsOccurred; + } + @Getter private final long messagesProcessed; @Getter diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKv.java new file mode 100644 index 0000000000..033413864c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/AttributeKv.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class AttributeKv { + private EntityId entityId; + private AttributeScope scope; + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonIgnoreProperties(ignoreUnknown = true) + private AttributeKvEntry entry; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java index 4dbbc3a211..f54d63e471 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BasicKvEntry.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @@ -36,26 +37,31 @@ public abstract class BasicKvEntry implements KvEntry { return key; } + @JsonIgnore @Override public Optional getStrValue() { return Optional.ofNullable(null); } + @JsonIgnore @Override public Optional getLongValue() { return Optional.ofNullable(null); } + @JsonIgnore @Override public Optional getBooleanValue() { return Optional.ofNullable(null); } + @JsonIgnore @Override public Optional getDoubleValue() { return Optional.ofNullable(null); } + @JsonIgnore @Override public Optional getJsonValue() { return Optional.ofNullable(null); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java index 89758673bf..c23237aee9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/BooleanDataEntry.java @@ -15,13 +15,17 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; import java.util.Optional; public class BooleanDataEntry extends BasicKvEntry { private final Boolean value; - public BooleanDataEntry(String key, Boolean value) { + @JsonCreator + public BooleanDataEntry(@JsonProperty("key") String key, @JsonProperty("value") Boolean value) { super(key); this.value = value; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java index 45d918c7ef..1bdbda85d4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/DoubleDataEntry.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; import java.util.Optional; @@ -22,7 +25,8 @@ public class DoubleDataEntry extends BasicKvEntry { private final Double value; - public DoubleDataEntry(String key, Double value) { + @JsonCreator + public DoubleDataEntry(@JsonProperty("key") String key, @JsonProperty("value") Double value) { super(key); this.value = value; } @@ -62,7 +66,7 @@ public class DoubleDataEntry extends BasicKvEntry { "value=" + value + "} " + super.toString(); } - + @Override public String getValueAsString() { return Double.toString(value); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java index 75e3fc3d2b..7fe3abc26b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/JsonDataEntry.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; import java.util.Optional; @@ -22,7 +25,8 @@ public class JsonDataEntry extends BasicKvEntry { private final String value; - public JsonDataEntry(String key, String value) { + @JsonCreator + public JsonDataEntry(@JsonProperty("key") String key, @JsonProperty("value") String value) { super(key); this.value = value; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/KvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/KvEntry.java index 7f3ed0052f..0dda760adf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/KvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/KvEntry.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonIgnore; + import java.io.Serializable; import java.util.Optional; @@ -39,6 +41,7 @@ public interface KvEntry extends Serializable { Optional getJsonValue(); + @JsonIgnore String getValueAsString(); Object getValue(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/LatestTsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/LatestTsKv.java new file mode 100644 index 0000000000..39be7ab76d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/LatestTsKv.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class LatestTsKv { + private EntityId entityId; + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonIgnoreProperties(ignoreUnknown = true) + private TsKvEntry entry; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/LongDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/LongDataEntry.java index cc61bdf255..a82625149e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/LongDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/LongDataEntry.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; import java.util.Optional; @@ -22,7 +25,8 @@ public class LongDataEntry extends BasicKvEntry { private final Long value; - public LongDataEntry(String key, Long value) { + @JsonCreator + public LongDataEntry(@JsonProperty("key") String key, @JsonProperty("value") Long value) { super(key); this.value = value; } @@ -62,7 +66,7 @@ public class LongDataEntry extends BasicKvEntry { "value=" + value + "} " + super.toString(); } - + @Override public String getValueAsString() { return Long.toString(value); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java index f00233f28c..9e9b1ee1f0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/StringDataEntry.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.common.data.kv; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; import java.util.Optional; @@ -24,7 +27,8 @@ public class StringDataEntry extends BasicKvEntry { private final String value; - public StringDataEntry(String key, String value) { + @JsonCreator + public StringDataEntry(@JsonProperty("key") String key, @JsonProperty("value") String value) { super(key); this.value = value; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKv.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKv.java new file mode 100644 index 0000000000..b25ce5beee --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKv.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.kv; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class TsKv { + private EntityId entityId; + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonIgnoreProperties(ignoreUnknown = true) + private TsKvEntry entry; + private long ttl; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 72883c55ef..bf278fff9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -24,7 +24,7 @@ import java.util.Collection; import java.util.List; import java.util.UUID; -public interface Dao { +public interface Dao extends ObjectDao { List find(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ObjectDao.java b/dao/src/main/java/org/thingsboard/server/dao/ObjectDao.java new file mode 100644 index 0000000000..358c16902f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ObjectDao.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ObjectDao { // TODO: choose a better name + + T save(TenantId tenantId, T entity); + + default ObjectType getType() { + return null; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index c295108a47..6452cedcac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -public interface TenantEntityDao { +public interface TenantEntityDao extends ObjectDao { default Long countByTenantId(TenantId tenantId) { throw new UnsupportedOperationException(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 5527d17add..7add16190c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -21,7 +21,9 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKv; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.dao.ObjectDao; import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; @@ -33,7 +35,7 @@ import java.util.UUID; /** * @author Andrew Shvayka */ -public interface AttributesDao { +public interface AttributesDao extends ObjectDao { Optional find(TenantId tenantId, EntityId entityId, AttributeScope attributeScope, String attributeKey); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 2ebcf046a6..b3b3031528 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.entity.CachedVersionedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; @@ -87,6 +88,9 @@ public class DeviceProfileServiceImpl extends CachedVersionedEntityService> daos = new EnumMap<>(EntityType.class); + private final Map> objectDaos = new EnumMap<>(ObjectType.class); + private final Map> entityDaos = new EnumMap<>(EntityType.class); - private EntityDaoRegistry(List> daos) { + private EntityDaoRegistry(List> daos) { daos.forEach(dao -> { - EntityType entityType = dao.getEntityType(); - if (entityType != null) { - this.daos.put(entityType, dao); + if (dao instanceof Dao entityDao) { + EntityType entityType = entityDao.getEntityType(); + if (entityType != null) { + entityDaos.put(entityType, entityDao); + } + } + ObjectType objectType = dao.getType(); + if (objectType != null) { + objectDaos.put(objectType, dao); } }); } public Dao getDao(EntityType entityType) { - Dao dao = (Dao) daos.get(entityType); + Dao dao = (Dao) entityDaos.get(entityType); if (dao == null) { throw new IllegalArgumentException("Missing dao for entity type " + entityType); } return dao; } + public ObjectDao getDao(ObjectType objectType) { + ObjectDao dao = (ObjectDao) objectDaos.get(objectType); + if (dao == null) { + throw new IllegalArgumentException("Missing dao for object type " + objectType); + } + return dao; + } + + public TenantEntityDao getTenantEntityDao(ObjectType objectType) { + ObjectDao dao = objectDaos.get(objectType); + if (!(dao instanceof TenantEntityDao)) { + throw new IllegalArgumentException("Missing tenant entity dao for entity type " + objectType); + } + return (TenantEntityDao) dao; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java index 7494ad2532..65dae4e853 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/EventDao.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.event.EventFilter; import org.thingsboard.server.common.data.event.EventType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.ObjectDao; import java.util.List; import java.util.UUID; @@ -28,7 +29,7 @@ import java.util.UUID; /** * The Interface EventDao. */ -public interface EventDao { +public interface EventDao extends ObjectDao { /** * Save or update event object async diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 23324eccb5..6266ddf49c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -281,6 +281,8 @@ public class ModelConstants { public static final String ALARM_COMMENT_TYPE = "type"; public static final String ALARM_COMMENT_COMMENT = "comment"; + public static final String ALARM_TYPES_TABLE_NAME = "alarm_types"; + /** * Entity relation constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java new file mode 100644 index 0000000000..60f5232142 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeCompositeKey.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class AlarmTypeCompositeKey implements Serializable { + + private UUID tenantId; + private String type; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java new file mode 100644 index 0000000000..729d7a44f0 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AlarmTypeEntity.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.model.sql; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Table; +import lombok.Data; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.model.ToData; + +import java.util.UUID; + +@Data +@Entity +@Table(name = ModelConstants.ALARM_TYPES_TABLE_NAME) +@IdClass(AlarmTypeCompositeKey.class) +public class AlarmTypeEntity implements ToData { + + @Id + @Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) + private UUID tenantId; + + @Id + @Column(name = ModelConstants.ALARM_TYPE_PROPERTY, nullable = false) + private String type; + + public AlarmTypeEntity() {} + + public AlarmTypeEntity(AlarmType alarmType) { + setTenantId(alarmType.getTenantId().getId()); + setType(alarmType.getType()); + } + + public AlarmTypeEntity(UUID tenantId, String type) { + this.tenantId = tenantId; + this.type = type; + } + + @Override + public AlarmType toData() { + AlarmType alarmType = new AlarmType(); + alarmType.setTenantId(TenantId.fromUUID(tenantId)); + alarmType.setType(type); + return alarmType; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 16d8517b6d..45f024ac91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -235,24 +235,41 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = - new PaginatedRemover<>() { + private final PaginatedRemover tenantOtaPackageRemover = new PaginatedRemover<>() { + + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { + deleteOtaPackage(tenantId, entity.getId()); + } + }; - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); - } + private final PaginatedRemover deviceProfileOtaPackageRemover = new PaginatedRemover<>() { - @Override - protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { - deleteOtaPackage(tenantId, entity.getId()); - } - }; + @Override + protected PageData findEntities(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + return otaPackageInfoDao.findOtaPackageIdsByDeviceProfileId(tenantId, deviceProfileId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, OtaPackageId id) { + deleteOtaPackage(tenantId, id); + } + }; @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java index 5802c70796..451315c1bf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageInfoDao.java @@ -30,6 +30,8 @@ public interface OtaPackageInfoDao extends Dao { PageData findOtaPackageInfoByTenantIdAndDeviceProfileIdAndTypeAndHasData(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType otaPackageType, PageLink pageLink); + PageData findOtaPackageIdsByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); + boolean isOtaPackageUsed(OtaPackageId otaPackageId, OtaPackageType otaPackageType, DeviceProfileId deviceProfileId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index c061faed5e..14fd9e0695 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -21,13 +21,14 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.dao.ObjectDao; import java.util.List; /** * Created by ashvayka on 25.04.17. */ -public interface RelationDao { +public interface RelationDao extends ObjectDao { List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java index 761a99c621..1f08f3635e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java @@ -19,7 +19,6 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.BaseData; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.dao.device.DeviceProfileDao; @@ -52,8 +51,7 @@ public abstract class BaseOtaPackageDataValidator> extends } if (otaPackageInfo.getDeviceProfileId() != null) { - DeviceProfile deviceProfile = getDeviceProfileDao().findById(otaPackageInfo.getTenantId(), otaPackageInfo.getDeviceProfileId().getId()); - if (deviceProfile == null) { + if (!deviceProfileDao.existsById(otaPackageInfo.getTenantId(), otaPackageInfo.getDeviceProfileId().getId())) { throw new DataValidationException("OtaPackage is referencing to non-existent device profile!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java index 1a24ff6abb..86782d1059 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmCommentRepository.java @@ -28,12 +28,12 @@ import java.util.UUID; public interface AlarmCommentRepository extends JpaRepository { @Query(value = "SELECT new org.thingsboard.server.dao.model.sql.AlarmCommentInfoEntity(a, u.firstName, u.lastName, u.email) FROM AlarmCommentEntity a " + - "LEFT JOIN UserEntity u on u.id = a.userId " + - "WHERE a.alarmId = :alarmId ", + "LEFT JOIN UserEntity u on u.id = a.userId " + + "WHERE a.alarmId = :alarmId ", countQuery = "" + - "SELECT count(a) " + - "FROM AlarmCommentEntity a " + - "WHERE a.alarmId = :alarmId ") + "SELECT count(a) " + + "FROM AlarmCommentEntity a " + + "WHERE a.alarmId = :alarmId ") Page findAllByAlarmId(@Param("alarmId") UUID alarmId, Pageable pageable); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java new file mode 100644 index 0000000000..d2ece1b94f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmTypeRepository.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.dao.model.sql.AlarmTypeCompositeKey; +import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; +import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; +import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; + +import java.util.List; +import java.util.UUID; + +public interface AlarmTypeRepository extends JpaRepository { + + Page findByTenantId(UUID tenantId, Pageable pageable); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java index 42c9d68b01..74ac3647b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmCommentDao.java @@ -22,6 +22,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.alarm.AlarmCommentInfo; import org.thingsboard.server.common.data.id.AlarmId; @@ -92,4 +93,9 @@ public class JpaAlarmCommentDao extends JpaPartitionedAbstractDao implements A return EntityType.ALARM; } + @Override + public ObjectType getType() { + return ObjectType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java new file mode 100644 index 0000000000..85f3d88f0a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmTypeDao.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.alarm; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.alarm.AlarmType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; +import org.thingsboard.server.dao.util.SqlDao; + +@Component +@SqlDao +public class JpaAlarmTypeDao implements TenantEntityDao { + + @Autowired + private AlarmTypeRepository alarmTypeRepository; + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(alarmTypeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "tenantId", "type"))); + } + + @Override + public AlarmType save(TenantId tenantId, AlarmType alarmType) { + return alarmTypeRepository.save(new AlarmTypeEntity(alarmType)).toData(); + } + + @Override + public ObjectType getType() { + return ObjectType.ALARM_TYPE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java index 796fe63da8..ac6f26d00a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaEntityAlarmDao.java @@ -17,12 +17,14 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.alarm.EntityAlarm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; import org.thingsboard.server.dao.util.SqlDao; @Component @@ -37,4 +39,14 @@ public class JpaEntityAlarmDao implements TenantEntityDao { return DaoUtil.toPageData(entityAlarmRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "entityId", "alarmId"))); } + @Override + public EntityAlarm save(TenantId tenantId, EntityAlarm entityAlarm) { + return entityAlarmRepository.save(new EntityAlarmEntity(entityAlarm)).toData(); + } + + @Override + public ObjectType getType() { + return ObjectType.ENTITY_ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 4e99fb57e4..fad66bb79b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -23,6 +23,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; @@ -311,4 +312,9 @@ public class JpaAssetDao extends JpaAbstractDao implements A return EntityType.ASSET; } + @Override + public ObjectType getType() { + return ObjectType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java index eeab5a338a..db896a72aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.asset.AssetProfileInfo; import org.thingsboard.server.common.data.edqs.fields.AssetProfileFields; @@ -156,4 +157,9 @@ public class JpaAssetProfileDao extends JpaAbstractDao imp return EntityType.CUSTOMER; } + @Override + public ObjectType getType() { + return ObjectType.CUSTOMER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 2d796c6917..75b97beaa8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.DashboardFields; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; @@ -107,4 +108,9 @@ public class JpaDashboardDao extends JpaAbstractDao return EntityType.DASHBOARD; } + @Override + public ObjectType getType() { + return ObjectType.DASHBOARD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java index 7b938ea274..f3e49ac5d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceCredentialsRepository.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.DeviceCredentialsEntity; +import org.thingsboard.server.dao.model.sql.UserCredentialsEntity; import java.util.UUID; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java index 7445d058d4..a86f89351e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceCredentialsDao.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -75,4 +77,9 @@ public class JpaDeviceCredentialsDao extends JpaAbstractDao implement return EntityType.DEVICE; } + @Override + public ObjectType getType() { + return ObjectType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index ebd8c78ed0..66cd154cf0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.fields.DeviceProfileFields; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -174,4 +175,9 @@ public class JpaDeviceProfileDao extends JpaAbstractDao findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index ea2abbba8a..4e7aebdfd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -25,15 +25,19 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeEventId; import org.thingsboard.server.common.data.id.EdgeId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.edge.EdgeEventDao; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.EdgeEventEntity; @@ -57,7 +61,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @SqlDao @RequiredArgsConstructor @Slf4j -public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao implements EdgeEventDao { +public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao implements EdgeEventDao, TenantEntityDao { private static final List SORT_ORDERS = Collections.singletonList(new SortOrder("seqId")); private final UUID systemTenantId = NULL_UUID; @@ -199,4 +203,14 @@ public class JpaBaseEdgeEventDao extends JpaPartitionedAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(edgeEventRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.EDGE_EVENT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 53ba6cf357..d0608184c8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -23,6 +23,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeInfo; import org.thingsboard.server.common.data.edqs.fields.EdgeFields; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.model.sql.EdgeEntity; import org.thingsboard.server.dao.model.sql.EdgeInfoEntity; @@ -46,7 +48,7 @@ import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; @Component @Slf4j @SqlDao -public class JpaEdgeDao extends JpaAbstractDao implements EdgeDao { +public class JpaEdgeDao extends JpaAbstractDao implements EdgeDao, TenantEntityDao { @Autowired private EdgeRepository edgeRepository; @@ -241,4 +243,9 @@ public class JpaEdgeDao extends JpaAbstractDao implements Edge return EntityType.EDGE; } + @Override + public ObjectType getType() { + return ObjectType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 44d8a09ff4..4076e1cdaf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.edqs.fields.EntityViewFields; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; @@ -235,4 +236,9 @@ public class JpaEntityViewDao extends JpaAbstractDao saveAsync(Event event) { log.debug("Save event [{}] ", event); + prepare(event); + return queue.add(event); + } + + private void prepare(Event event) { if (event.getId() == null) { UUID timeBased = Uuids.timeBased(); event.setId(new EventId(timeBased)); @@ -139,7 +147,6 @@ public class JpaBaseEventDao implements EventDao { } partitioningRepository.createPartitionIfNotExists(event.getType().getTable(), event.getCreatedTime(), partitionConfiguration.getPartitionSizeInMs(event.getType())); - return queue.add(event); } @Override @@ -478,5 +485,17 @@ public class JpaBaseEventDao implements EventDao { return repository; } + @SneakyThrows + @Override + public Event save(TenantId tenantId, Event event) { + prepare(event); + eventInsertRepository.save(List.of(event)); + return event; + } + + @Override + public ObjectType getType() { + return ObjectType.EVENT; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java index f302048ed8..5b89a72c12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/event/LifecycleEventRepository.java @@ -114,4 +114,5 @@ public interface LifecycleEventRepository extends EventRepository, JpaRepository { @Override - @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); - @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId AND e.e_type = 'IN' ORDER BY e.ts DESC LIMIT 1") + @Query(nativeQuery = true, value = "SELECT * FROM rule_node_debug_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId AND e.e_type = 'IN' ORDER BY e.ts DESC LIMIT 1") Optional findLatestDebugRuleNodeInEvent(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId); @Override @@ -151,4 +151,5 @@ public interface RuleNodeDebugEventRepository extends EventRepository, JpaRepository { @Override - @Query(nativeQuery = true, value = "SELECT * FROM stats_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") + @Query(nativeQuery = true, value = "SELECT * FROM stats_event e WHERE e.tenant_id = :tenantId AND e.entity_id = :entityId ORDER BY e.ts DESC LIMIT :limit") List findLatestEvents(@Param("tenantId") UUID tenantId, @Param("entityId") UUID entityId, @Param("limit") int limit); @Query("SELECT e FROM StatisticsEventEntity e WHERE " + @@ -117,4 +117,5 @@ public interface StatisticsEventRepository extends EventRepository implements QrCodeSettingsDao { +public class JpaQrCodeSettingsDao extends JpaAbstractDao implements QrCodeSettingsDao, TenantEntityDao { @Autowired private QrCodeSettingsRepository qrCodeSettingsRepository; @@ -49,6 +56,11 @@ public class JpaQrCodeSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(mobileAppSettingsRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return QrCodeSettingsEntity.class; @@ -58,4 +70,10 @@ public class JpaQrCodeSettingsDao extends JpaAbstractDao getRepository() { return qrCodeSettingsRepository; } + + @Override + public ObjectType getType() { + return ObjectType.MOBILE_APP_SETTINGS; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/mobile/QrCodeSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/mobile/QrCodeSettingsRepository.java index 99829dfc57..f218f50c27 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/mobile/QrCodeSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/mobile/QrCodeSettingsRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.mobile; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -33,4 +35,7 @@ public interface QrCodeSettingsRepository extends JpaRepository findAllByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java index 739423ee70..998b822b9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRuleDao.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.id.TenantId; @@ -122,4 +123,9 @@ public class JpaNotificationRuleDao extends JpaAbstractDao findOtaPackageIdsByDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink) { + return DaoUtil.pageToPageData(otaPackageInfoRepository.findIdsByDeviceProfileId(deviceProfileId.getId(), DaoUtil.toPageable(pageLink))) + .mapData(OtaPackageId::new); + } + @Override public boolean isOtaPackageUsed(OtaPackageId otaPackageId, OtaPackageType otaPackageType, DeviceProfileId deviceProfileId) { return otaPackageInfoRepository.isOtaPackageUsed(otaPackageId.getId(), deviceProfileId.getId(), otaPackageType.name()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/OtaPackageInfoRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/OtaPackageInfoRepository.java index fe7e2bf015..525e67e2d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/OtaPackageInfoRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/OtaPackageInfoRepository.java @@ -45,6 +45,10 @@ public interface OtaPackageInfoRepository extends JpaRepository findIdsByDeviceProfileId(@Param("deviceProfileId") UUID deviceProfileId, + Pageable pageable); + @Query("SELECT new OtaPackageInfoEntity(f.id, f.createdTime, f.tenantId, f.deviceProfileId, f.type, f.title, f.version, f.tag, f.url, f.fileName, f.contentType, f.checksumAlgorithm, f.checksum, f.dataSize, f.additionalInfo, CASE WHEN (f.data IS NOT NULL OR f.url IS NOT NULL) THEN true ELSE false END) FROM OtaPackageEntity f WHERE f.id = :id") OtaPackageInfoEntity findOtaPackageInfoById(@Param("id") UUID id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index 566e4eaddc..006d6cc1e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -98,4 +99,9 @@ public class JpaQueueDao extends JpaAbstractDao implements Q return EntityType.QUEUE; } + @Override + public ObjectType getType() { + return ObjectType.QUEUE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 7417418f54..a69830f7ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -293,4 +294,16 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple public List findRuleNodeToRuleChainRelations(RuleChainType ruleChainType, int limit) { return DaoUtil.convertDataList(relationRepository.findRuleNodeToRuleChainRelations(ruleChainType, PageRequest.of(0, limit))); } + + @Override + public EntityRelation save(TenantId tenantId, EntityRelation relation) { + saveRelation(tenantId, relation); + return relation; + } + + @Override + public ObjectType getType() { + return ObjectType.RELATION; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index 6cce9d76c2..82977d46bb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -20,6 +20,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceSubType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TbResourceId; @@ -146,4 +147,9 @@ public class JpaTbResourceDao extends JpaAbstractDao implements RpcDao, return EntityType.RPC; } + @Override + public ObjectType getType() { + return ObjectType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 77044d41dc..ee4b13fd22 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -22,6 +22,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edqs.fields.RuleChainFields; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -151,4 +152,9 @@ public class JpaRuleChainDao extends JpaAbstractDao return EntityType.RULE_CHAIN; } + @Override + public ObjectType getType() { + return ObjectType.RULE_CHAIN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 8ecce0672f..5c65e0b745 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -107,9 +108,19 @@ public class JpaRuleNodeDao extends JpaAbstractDao imp ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); } + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(ruleNodeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public EntityType getEntityType() { return EntityType.RULE_NODE; } + @Override + public ObjectType getType() { + return ObjectType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java index 9016dc346d..32fd37aae4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeStateDao.java @@ -20,10 +20,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleNodeState; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.RuleNodeStateEntity; import org.thingsboard.server.dao.rule.RuleNodeStateDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; @@ -34,7 +37,7 @@ import java.util.UUID; @Slf4j @Component @SqlDao -public class JpaRuleNodeStateDao extends JpaAbstractDao implements RuleNodeStateDao { +public class JpaRuleNodeStateDao extends JpaAbstractDao implements RuleNodeStateDao, TenantEntityDao { @Autowired private RuleNodeStateRepository ruleNodeStateRepository; @@ -70,4 +73,15 @@ public class JpaRuleNodeStateDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(ruleNodeStateRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.RULE_NODE_STATE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index 74d482126c..8d77eda30f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -63,4 +63,7 @@ public interface RuleNodeRepository extends JpaRepository @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") void deleteByIdIn(@Param("ids") List ids); + @Query("SELECT n FROM RuleNodeEntity n WHERE n.ruleChainId IN (SELECT rc.id FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeStateRepository.java index b9c08294ba..83af9cbbde 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeStateRepository.java @@ -35,4 +35,9 @@ public interface RuleNodeStateRepository extends JpaRepository findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 68ce5e9d22..a8b391e365 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -21,6 +21,10 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -77,4 +81,9 @@ public class JpaAdminSettingsDao extends JpaAbstractDao implement return tenantRepository.findNextBatch(id, Limit.of(batchSize)); } + @Override + public ObjectType getType() { + return ObjectType.TENANT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java index 4f9ff5d222..9afd06ad1c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserAuthSettingsDao.java @@ -18,8 +18,13 @@ package org.thingsboard.server.dao.sql.user; import lombok.RequiredArgsConstructor; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserAuthSettings; +import org.thingsboard.server.common.data.security.model.mfa.account.AccountTwoFaSettings; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; @@ -46,6 +51,18 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + PageData data = DaoUtil.toPageData(repository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + data.getData().forEach(settings -> { + AccountTwoFaSettings twoFaSettings = settings.getTwoFaSettings(); + if (twoFaSettings != null && twoFaSettings.getConfigs() != null) { + twoFaSettings.getConfigs().values().forEach(config -> config.setSerializeHiddenFields(true)); + } + }); + return data; + } + @Override protected Class getEntityClass() { return UserAuthSettingsEntity.class; @@ -56,4 +73,9 @@ public class JpaUserAuthSettingsDao extends JpaAbstractDao implements User return EntityType.USER; } + @Override + public ObjectType getType() { + return ObjectType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java index 646b11975e..f4202470ab 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserSettingsDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.user; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; @@ -73,4 +74,9 @@ public class JpaUserSettingsDao implements UserSettingsDao, TenantEntityDao { + + @Autowired + private WidgetsBundleWidgetRepository widgetsBundleWidgetRepository; + + @Override + public WidgetsBundleWidget save(TenantId tenantId, WidgetsBundleWidget entity) { + return widgetsBundleWidgetRepository.save(new WidgetsBundleWidgetEntity(entity)).toData(); + } + + @Override + public PageData findAllByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(widgetsBundleWidgetRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public ObjectType getType() { + return ObjectType.WIDGETS_BUNDLE_WIDGET; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java index a4d19d5c80..421995d851 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleWidgetRepository.java @@ -15,7 +15,11 @@ */ package org.thingsboard.server.dao.sql.widget; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetCompositeKey; import org.thingsboard.server.dao.model.sql.WidgetsBundleWidgetEntity; @@ -26,4 +30,7 @@ public interface WidgetsBundleWidgetRepository extends JpaRepository findAllByWidgetsBundleId(UUID widgetsBundleId); + @Query("SELECT w FROM WidgetsBundleWidgetEntity w WHERE w.widgetsBundleId IN (SELECT b.id FROM WidgetsBundleEntity b WHERE b.tenantId = :tenantId)") + Page findByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index 1c5d554350..b9022dffb4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -22,6 +22,7 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Direction; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -48,9 +49,9 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; import java.util.function.Function; -@SuppressWarnings("UnstableApiUsage") @Slf4j public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSqlTimeseriesDao implements TimeseriesDao { @@ -144,7 +145,29 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq } } - ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { + @Override + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, String key, Consumer processor) { + Integer keyId = keyDictionaryDao.getOrSaveKeyId(key); + long currentTs = System.currentTimeMillis(); + PageRequest batchRequest = PageRequest.ofSize(256).withSort(Sort.by(Direction.ASC, "ts")); + + while (true) { + List tsKvEntities = tsKvRepository.findAllWithLimit(entityId.getId(), keyId, 0, currentTs, batchRequest); + if (tsKvEntities.isEmpty()) { + break; + } + for (TsKvEntity tsKvEntity : tsKvEntities) { + tsKvEntity.setStrKey(key); + TsKvEntry tsKvEntry = tsKvEntity.toData(); + processor.accept(tsKvEntry); + } + + batchRequest = batchRequest.next(); + } + return Futures.immediateVoidFuture(); + } + + private ReadTsKvQueryResult findAllAsyncWithLimit(EntityId entityId, ReadTsKvQuery query) { Integer keyId = keyDictionaryDao.getOrSaveKeyId(query.getKey()); List tsKvEntities = tsKvRepository.findAllWithLimit( entityId.getId(), @@ -199,4 +222,5 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq throw new IllegalArgumentException("Not supported aggregation type: " + aggregation); } } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 6913aa8ea6..9dd5d7202c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.ScheduledLogExecutorComponent; +import org.thingsboard.server.dao.timeseries.TimeseriesDao; import java.sql.Connection; import java.sql.PreparedStatement; @@ -40,7 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j -public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements AggregationTimeseriesDao { +public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseriesDao implements TimeseriesDao, AggregationTimeseriesDao { protected static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java index fad1e1bf91..4d609ac7d8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/sql/JpaSqlTimeseriesDao.java @@ -18,14 +18,17 @@ package org.thingsboard.server.dao.sqlts.sql; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.TsKv; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; @@ -167,4 +170,17 @@ public class JpaSqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDao private static long toMills(LocalDateTime time) { return time.toInstant(ZoneOffset.UTC).toEpochMilli(); } + + @SneakyThrows + @Override + public TsKv save(TenantId tenantId, TsKv tsKv) { + save(tenantId, tsKv.getEntityId(), tsKv.getEntry(), tsKv.getTtl()).get(30, TimeUnit.SECONDS); + return tsKv; + } + + @Override + public ObjectType getType() { + return ObjectType.TS_KV; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index 7e5feac876..b9240910cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -20,12 +20,16 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.SneakyThrows; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -33,6 +37,7 @@ import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKv; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; @@ -53,6 +58,8 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; import java.util.function.Function; @Component @@ -174,6 +181,28 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } } + @Override + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, String key, Consumer processor) { + Integer keyId = keyDictionaryDao.getOrSaveKeyId(key); + long currentTs = System.currentTimeMillis(); + PageRequest batchRequest = PageRequest.ofSize(256).withSort(Sort.by(Sort.Direction.ASC, "ts")); + + while (true) { + List tsKvEntities = tsKvRepository.findAllWithLimit(entityId.getId(), keyId, 0, currentTs, batchRequest); + if (tsKvEntities.isEmpty()) { + break; + } + for (TimescaleTsKvEntity tsKvEntity : tsKvEntities) { + tsKvEntity.setStrKey(key); + TsKvEntry tsKvEntry = tsKvEntity.toData(); + processor.accept(tsKvEntry); + } + + batchRequest = batchRequest.next(); + } + return Futures.immediateVoidFuture(); + } + @Override public void cleanup(long systemTtl) { super.cleanup(systemTtl); @@ -246,4 +275,16 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements } } + @SneakyThrows + @Override + public TsKv save(TenantId tenantId, TsKv tsKv) { + save(tenantId, tsKv.getEntityId(), tsKv.getEntry(), tsKv.getTtl()).get(30, TimeUnit.SECONDS); + return tsKv; + } + + @Override + public ObjectType getType() { + return ObjectType.TS_KV; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index f7bd64bef8..63d9047dcf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -28,16 +28,22 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import jakarta.annotation.Nullable; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; import jakarta.annotation.Nullable; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.Getter; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.core.env.Profiles; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; @@ -48,9 +54,9 @@ import org.thingsboard.server.common.data.kv.IntervalType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKv; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntryAggWrapper; -import org.thingsboard.server.common.data.kv.TsKvQuery; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.nosql.TbResultSetFuture; @@ -70,6 +76,7 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; import java.util.stream.Collectors; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -77,7 +84,6 @@ import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; /** * @author Andrew Shvayka */ -@SuppressWarnings("UnstableApiUsage") @Component @Slf4j @NoSqlTsDao @@ -237,11 +243,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); + final ListenableFuture> partitionsListFuture = getPartitions(tenantId, entityId, query.getKey(), query.getStartTs(), query.getEndTs()); Futures.addCallback(partitionsListFuture, new FutureCallback>() { @Override @@ -252,7 +255,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); + log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), query.getStartTs(), query.getEndTs(), t); } }, readResultsProcessingExecutor); return resultFuture; @@ -278,7 +281,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } endTs = Math.min(endTs, endPeriod); ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, endTs - startTs, 1, query.getAggregation(), query.getOrder()); - futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); + futures.add(findAndAggregateAsync(tenantId, entityId, subQuery)); startPeriod = endTs; } ListenableFuture>> future = Futures.allAsList(futures); @@ -306,27 +309,55 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } } + @Override + public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, String key, Consumer processor) { + ListenableFuture> partitionsFuture = getPartitions(tenantId, entityId, key, 0, System.currentTimeMillis()); + SettableFuture> resultFuture = SettableFuture.create(); + Futures.addCallback(partitionsFuture, new FutureCallback<>() { + @Override + public void onSuccess(List partitions) { + TsKvQueryCursor cursor = TsKvQueryCursor.builder() + .entityType(entityId.getEntityType().name()) + .entityId(entityId.getId()) + .baseQuery(new BaseReadTsKvQuery(key, 0, System.currentTimeMillis(), Integer.MAX_VALUE, "ASC")) + .partitions(partitions) + .dataProcessor(tsKvEntries -> tsKvEntries.forEach(processor)) + .build(); + findAllAsyncSequentiallyWithLimit(tenantId, cursor, resultFuture); + } + + @Override + public void onFailure(Throwable t) { + resultFuture.setException(t); + } + }, readResultsProcessingExecutor); + return Futures.transform(resultFuture, result -> null, MoreExecutors.directExecutor()); + } + @Override public void cleanup(long systemTtl) { //Cleanup by TTL is native for Cassandra } private ListenableFuture findAllAsyncWithLimit(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - final ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); - final SimpleListenableFuture> resultFuture = new SimpleListenableFuture<>(); - + ListenableFuture> partitionsListFuture = getPartitions(tenantId, entityId, query.getKey(), query.getStartTs(), query.getEndTs()); + SettableFuture> resultFuture = SettableFuture.create(); Futures.addCallback(partitionsListFuture, new FutureCallback<>() { @Override public void onSuccess(@Nullable List partitions) { - TsKvQueryCursor cursor = new TsKvQueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitions); + TsKvQueryCursor cursor = TsKvQueryCursor.builder() + .entityType(entityId.getEntityType().name()) + .entityId(entityId.getId()) + .baseQuery(query) + .partitions(partitions) + .build(); findAllAsyncSequentiallyWithLimit(tenantId, cursor, resultFuture); } @Override public void onFailure(Throwable t) { log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), toPartitionTs(query.getStartTs()), toPartitionTs(query.getEndTs()), t); + resultFuture.setException(t); } }, readResultsProcessingExecutor); @@ -344,7 +375,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } - private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SimpleListenableFuture> resultFuture) { + private void findAllAsyncSequentiallyWithLimit(TenantId tenantId, final TsKvQueryCursor cursor, final SettableFuture> resultFuture) { if (cursor.isFull() || !cursor.hasNextPartition()) { resultFuture.set(cursor.getData()); } else { @@ -378,7 +409,8 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch data for query {}-{}", stmt, t); + log.error("[{}] Failed to fetch data for query {}", tenantId, stmt, t); + resultFuture.setException(t); } }, readResultsProcessingExecutor); @@ -388,19 +420,20 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch data for query {}-{}", stmt, t); + log.error("[{}] Failed to fetch data for query {}", tenantId, stmt, t); + resultFuture.setException(t); } }, readResultsProcessingExecutor); } } - private ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query, long minPartition, long maxPartition) { + private ListenableFuture> findAndAggregateAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { final Aggregation aggregation = query.getAggregation(); final String key = query.getKey(); final long startTs = query.getStartTs(); final long endTs = query.getEndTs(); final long ts = startTs + (endTs - startTs) / 2; - ListenableFuture> partitionsListFuture = getPartitionsFuture(tenantId, query, entityId, minPartition, maxPartition); + ListenableFuture> partitionsListFuture = getPartitions(tenantId, entityId, key, startTs, endTs); ListenableFuture> aggregationChunks = Futures.transformAsync(partitionsListFuture, getFetchChunksAsyncFunction(tenantId, entityId, key, aggregation, startTs, endTs), readResultsProcessingExecutor); @@ -415,14 +448,17 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD readResultsProcessingExecutor); } - private ListenableFuture> getPartitionsFuture(TenantId tenantId, TsKvQuery query, EntityId entityId, long minPartition, long maxPartition) { + private ListenableFuture> getPartitions(TenantId tenantId, EntityId entityId, String key, long startTs, long endTs) { if (isFixedPartitioning()) { //no need to fetch partitions from DB return Futures.immediateFuture(FIXED_PARTITION); } + + long minPartition = toPartitionTs(startTs); + long maxPartition = toPartitionTs(endTs); if (!isUseTsKeyValuePartitioningOnRead()) { return Futures.immediateFuture(calculatePartitions(minPartition, maxPartition)); } - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); + TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, key, minPartition, maxPartition); return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); } @@ -510,6 +546,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD public void onFailure(Throwable t) { } + } private long computeTtl(long ttl) { @@ -798,4 +835,19 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD } return fetchStmts; } + + @SneakyThrows + @Override + public TsKv save(TenantId tenantId, TsKv tsKv) { + ListenableFuture partitionFuture = savePartition(tenantId, tsKv.getEntityId(), tsKv.getEntry().getTs(), tsKv.getEntry().getKey()); + ListenableFuture tsKvFuture = save(tenantId, tsKv.getEntityId(), tsKv.getEntry(), tsKv.getTtl()); + Futures.allAsList(partitionFuture, tsKvFuture).get(30, TimeUnit.SECONDS); + return tsKv; + } + + @Override + public ObjectType getType() { + return ObjectType.TS_KV; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 54a7e68725..f50c859257 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -23,15 +23,18 @@ import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.LatestTsKv; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -48,6 +51,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.TimeUnit; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -239,4 +243,17 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes } return findAllLatestStmt; } + + @SneakyThrows + @Override + public LatestTsKv save(TenantId tenantId, LatestTsKv latestTsKv) { + saveLatest(tenantId, latestTsKv.getEntityId(), latestTsKv.getEntry()).get(30, TimeUnit.SECONDS); + return latestTsKv; + } + + @Override + public ObjectType getType() { + return ObjectType.LATEST_TS_KV; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index 2e4e247c24..8881f4d9c2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKv; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.dao.ObjectDao; import java.util.List; +import java.util.function.Consumer; /** * @author Andrew Shvayka */ -public interface TimeseriesDao { +public interface TimeseriesDao extends ObjectDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries); @@ -39,4 +42,7 @@ public interface TimeseriesDao { ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); void cleanup(long systemTtl); + + ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, String key, Consumer processor); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 32479301ae..02ade8f2e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -20,8 +20,10 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.LatestTsKv; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; +import org.thingsboard.server.dao.ObjectDao; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; @@ -30,7 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -public interface TimeseriesLatestDao { +public interface TimeseriesLatestDao extends ObjectDao { /** * Optional TsKvEntry if the value is present in the DB diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java index ecc541d29f..53b8463ab5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TsKvQueryCursor.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.timeseries; +import lombok.Builder; import lombok.Getter; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -22,6 +23,7 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.function.Consumer; import static org.thingsboard.server.dao.timeseries.CassandraBaseTimeseriesDao.DESC_ORDER; @@ -32,18 +34,21 @@ public class TsKvQueryCursor extends QueryCursor { @Getter private final List data; + private final Consumer> dataProcessor; // if non-null, "data" will be empty @Getter private final String orderBy; private int partitionIndex; private int currentLimit; - public TsKvQueryCursor(String entityType, UUID entityId, ReadTsKvQuery baseQuery, List partitions) { + @Builder + public TsKvQueryCursor(String entityType, UUID entityId, ReadTsKvQuery baseQuery, List partitions, Consumer> dataProcessor) { super(entityType, entityId, baseQuery, partitions); this.orderBy = baseQuery.getOrder(); this.partitionIndex = isDesc() ? partitions.size() - 1 : 0; this.data = new ArrayList<>(); this.currentLimit = baseQuery.getLimit(); + this.dataProcessor = dataProcessor; } @Override @@ -72,10 +77,15 @@ public class TsKvQueryCursor extends QueryCursor { public void addData(List newData) { currentLimit -= newData.size(); - data.addAll(newData); + if (dataProcessor == null) { + data.addAll(newData); + } else { + dataProcessor.accept(newData); + } } private boolean isDesc() { return orderBy.equals(DESC_ORDER); } + } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index f2a0bc26c1..d6b5019fdb 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -303,18 +303,6 @@ CREATE TABLE IF NOT EXISTS device_profile ( CONSTRAINT fk_default_edge_rule_chain_device_profile FOREIGN KEY (default_edge_rule_chain_id) REFERENCES rule_chain(id) ); -DO -$$ - BEGIN - IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_device_profile_ota_package') THEN - ALTER TABLE ota_package - ADD CONSTRAINT fk_device_profile_ota_package - FOREIGN KEY (device_profile_id) REFERENCES device_profile (id) - ON DELETE CASCADE; - END IF; - END; -$$; - -- We will use one-to-many relation in the first release and extend this feature in case of user requests -- CREATE TABLE IF NOT EXISTS device_profile_firmware ( -- device_profile_id uuid NOT NULL, diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java index 86313dfde9..05bebfeb90 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -21,16 +21,29 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ObjectDao; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.entity.EntityDaoRegistry; +import java.util.EnumSet; import java.util.List; +import java.util.Set; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; +import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; +import static org.thingsboard.server.common.data.ObjectType.EVENT; +import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; +import static org.thingsboard.server.common.data.ObjectType.RELATION; +import static org.thingsboard.server.common.data.ObjectType.TENANT; +import static org.thingsboard.server.common.data.ObjectType.TS_KV; @Slf4j @DaoSqlTest @@ -88,4 +101,27 @@ public class EntityDaoRegistryTest extends AbstractServiceTest { } } + @Test + public void givenAllTenantEntityDaos_whenFindAllByTenantId_thenOk() { + Set ignored = EnumSet.of(TENANT, RELATION, EVENT, ATTRIBUTE_KV, LATEST_TS_KV, TS_KV, AUDIT_LOG); + for (ObjectType type : ObjectType.values()) { + if (ignored.contains(type)) { + continue; + } + + TenantEntityDao dao = assertDoesNotThrow(() -> entityDaoRegistry.getTenantEntityDao(type)); + assertDoesNotThrow(() -> { + dao.findAllByTenantId(tenantId, new PageLink(100)); + }); + } + } + + @Test + public void givenAllObjectTypes_whenGetDao_thenPresent() { + for (ObjectType type : ObjectType.values()) { + ObjectDao dao = assertDoesNotThrow(() -> entityDaoRegistry.getDao(type)); + assertThat(dao).isNotNull(); + } + } + } diff --git a/pom.xml b/pom.xml index 64cdd63eaf..7d08fd159e 100755 --- a/pom.xml +++ b/pom.xml @@ -66,6 +66,7 @@ 2.16.1 1.3.1 1.10.0 + 1.26.1 5.3.1 5.2.4 4.5.14 @@ -1538,6 +1539,11 @@ commons-csv ${commons-csv.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + org.apache.httpcomponents.client5 httpclient5