Browse Source
* Implement findAllByTenantId for most entities * Implement export and import of whole tenant * Major updates for tenant export-import; more tests * Ts kv export/import; refactoring * Export-import results in Redis * Export cancellation, refactoring * Fixes and improvements * Fix ota package circular foreign key constraint * Minor refactoring * Mobile settings export import; refactoring * Minor refactoring * Fix CassandraBaseTimeseriesDaofeature/tenant-export
Viacheslav Klimov
1 year ago
committed by
GitHub
GitHub
116 changed files with 2727 additions and 95 deletions
@ -0,0 +1,341 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant; |
|||
|
|||
import jakarta.annotation.PostConstruct; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.exception.ExceptionUtils; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.cache.CacheManager; |
|||
import org.springframework.core.io.InputStreamResource; |
|||
import org.springframework.http.HttpHeaders; |
|||
import org.springframework.http.MediaType; |
|||
import org.springframework.http.ResponseEntity; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.CacheConstants; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.audit.AuditLog; |
|||
import org.thingsboard.server.common.data.event.Event; |
|||
import org.thingsboard.server.common.data.event.EventType; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.HasId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.kv.AttributeKv; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.LatestTsKv; |
|||
import org.thingsboard.server.common.data.kv.TsKv; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageDataIterable; |
|||
import org.thingsboard.server.common.data.page.TimePageLink; |
|||
import org.thingsboard.server.common.data.relation.EntityRelation; |
|||
import org.thingsboard.server.dao.TenantEntityDao; |
|||
import org.thingsboard.server.dao.attributes.AttributesDao; |
|||
import org.thingsboard.server.dao.audit.AuditLogDao; |
|||
import org.thingsboard.server.dao.entity.EntityDaoRegistry; |
|||
import org.thingsboard.server.dao.event.EventDao; |
|||
import org.thingsboard.server.dao.model.ModelConstants; |
|||
import org.thingsboard.server.dao.relation.RelationDao; |
|||
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; |
|||
import org.thingsboard.server.dao.tenant.TenantDao; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesDao; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesLatestDao; |
|||
import org.thingsboard.server.service.sync.tenant.util.DataWrapper; |
|||
import org.thingsboard.server.service.sync.tenant.util.Result; |
|||
import org.thingsboard.server.service.sync.tenant.util.ResultStore; |
|||
import org.thingsboard.server.service.sync.tenant.util.Storage; |
|||
import org.thingsboard.server.service.sync.tenant.util.TenantExportConfig; |
|||
|
|||
import java.util.Arrays; |
|||
import java.util.Collections; |
|||
import java.util.EnumSet; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.function.BiConsumer; |
|||
|
|||
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; |
|||
import static org.thingsboard.server.common.data.ObjectType.EVENT; |
|||
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.RELATION; |
|||
import static org.thingsboard.server.common.data.ObjectType.TENANT; |
|||
import static org.thingsboard.server.common.data.ObjectType.TS_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.values; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class TenantExportService { |
|||
|
|||
private final Storage storage; |
|||
private final EntityDaoRegistry entityDaoRegistry; |
|||
private final TenantDao tenantDao; |
|||
private final EventDao eventDao; |
|||
private final AuditLogDao auditLogDao; |
|||
private final AttributesDao attributesDao; |
|||
private final RelationDao relationDao; |
|||
private final TimeseriesLatestDao timeseriesLatestDao; |
|||
private final TimeseriesDao timeseriesDao; |
|||
private final SqlPartitioningRepository partitioningRepository; |
|||
private final CacheManager cacheManager; |
|||
@Value("${cache.specs.tenantExportResults.timeToLiveInMinutes:1440}") |
|||
private int resultsTtl; |
|||
private ResultStore<ObjectType> resultStore; |
|||
|
|||
private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tenant-export")); |
|||
|
|||
private Map<ObjectType, BiConsumer<TenantId, TenantExportConfig>> customExporters; |
|||
private Map<ObjectType, BiConsumer<TenantId, EntityId>> relatedEntitiesExporters; |
|||
|
|||
private static final Set<ObjectType> RELATED = EnumSet.of(EVENT, RELATION, ATTRIBUTE_KV, LATEST_TS_KV); |
|||
private static final Set<ObjectType> IGNORED = EnumSet.of(TENANT); |
|||
private static final List<ObjectType> EXPORTABLE = Arrays.stream(values()) |
|||
.filter(type -> !RELATED.contains(type) && !IGNORED.contains(type)) |
|||
.sorted().toList(); |
|||
|
|||
@PostConstruct |
|||
private void init() { |
|||
resultStore = ResultStore.<ObjectType>builder() |
|||
.name("Tenant export") |
|||
.ttlInMinutes(resultsTtl) |
|||
.persistFrequency(100) |
|||
.removalListener((tenantId, result) -> { |
|||
if (result.isDone() && result.isSuccess()) { |
|||
storage.cleanUpExportData(tenantId); |
|||
} |
|||
}) |
|||
.cacheName(CacheConstants.TENANT_EXPORT_RESULT_CACHE) |
|||
.cacheManager(cacheManager) |
|||
.build(); |
|||
relatedEntitiesExporters = Map.of( |
|||
RELATION, this::exportRelations, |
|||
EVENT, this::exportEvents, // todo: query by tenant
|
|||
ATTRIBUTE_KV, this::exportAttributes, |
|||
LATEST_TS_KV, this::exportLatestTelemetry |
|||
); |
|||
customExporters = Map.of( |
|||
TS_KV, this::exportTelemetry, |
|||
AUDIT_LOG, this::exportAuditLogs |
|||
); |
|||
} |
|||
|
|||
public UUID exportTenant(TenantExportConfig exportConfig) { |
|||
UUID tenantId = exportConfig.getTenantId(); |
|||
log.info("[{}] Exporting tenant", tenantId); |
|||
Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId); |
|||
if (tenant == null) { |
|||
throw new IllegalArgumentException("Tenant with id " + tenantId + " not found"); |
|||
} |
|||
executor.submit(() -> { |
|||
try { |
|||
exportTenant(tenant, exportConfig); |
|||
resultStore.update(tenantId, result -> { |
|||
result.setSuccess(true); |
|||
result.setDone(true); |
|||
}); |
|||
} catch (Throwable t) { |
|||
log.error("Failed to export tenant {}", tenant, t); |
|||
try { |
|||
resultStore.update(tenantId, result -> { |
|||
result.setError(ExceptionUtils.getStackTrace(t)); |
|||
result.getStats().clear(); |
|||
result.setDone(true); |
|||
}); |
|||
storage.cleanUpExportData(tenantId); |
|||
} catch (Exception e) { |
|||
log.error("Failed to handle export error", e); |
|||
} |
|||
} |
|||
}); |
|||
return tenantId; |
|||
} |
|||
|
|||
private void exportTenant(Tenant tenant, TenantExportConfig exportConfig) { |
|||
TenantId tenantId = tenant.getId(); |
|||
storage.init(tenantId.getId()); |
|||
|
|||
export(tenantId, TENANT, tenant, exportConfig); |
|||
for (ObjectType type : EXPORTABLE) { |
|||
if (exportConfig.getSkipped().contains(type)) { |
|||
continue; |
|||
} |
|||
|
|||
log.debug("[{}] Exporting {} entities", tenantId, type); |
|||
if (!customExporters.containsKey(type)) { |
|||
TenantEntityDao<?> dao = entityDaoRegistry.getTenantEntityDao(type); |
|||
var entities = new PageDataIterable<>(pageLink -> dao.findAllByTenantId(tenantId, pageLink), 100); |
|||
for (Object entity : entities) { |
|||
export(tenantId, type, entity, exportConfig); |
|||
} |
|||
} else { |
|||
customExporters.get(type).accept(tenantId, exportConfig); |
|||
} |
|||
resultStore.flush(tenantId.getId(), type); |
|||
} |
|||
resultStore.flush(tenantId.getId(), RELATED.toArray(ObjectType[]::new)); |
|||
|
|||
storage.archiveExportData(tenantId.getId()); |
|||
} |
|||
|
|||
private void export(TenantId tenantId, ObjectType type, Object entity, TenantExportConfig exportConfig) { |
|||
save(tenantId, type, entity); |
|||
if (entity instanceof HasId<?> hasId && hasId.getId() instanceof EntityId entityId) { |
|||
relatedEntitiesExporters.forEach((relatedEntityType, exporter) -> { |
|||
if (!exportConfig.getSkipped().contains(relatedEntityType)) { |
|||
exporter.accept(tenantId, entityId); |
|||
} |
|||
}); |
|||
} |
|||
} |
|||
|
|||
private void save(TenantId tenantId, ObjectType type, Object entity) { |
|||
DataWrapper dataWrapper = DataWrapper.of(entity); |
|||
|
|||
storage.save(tenantId.getId(), type, dataWrapper); |
|||
resultStore.report(tenantId.getId(), type); |
|||
log.trace("[{}][{}] Saved entity {}", tenantId, type, entity); |
|||
} |
|||
|
|||
private Map<Long, Long> getPartitions(String table) { |
|||
List<Long> partitionsStartTime = partitioningRepository.fetchPartitions(table).stream().sorted().toList(); |
|||
if (partitionsStartTime.isEmpty()) { |
|||
return Collections.emptyMap(); |
|||
} |
|||
|
|||
Map<Long, Long> partitions = new HashMap<>(); |
|||
for (int i = 0; i < partitionsStartTime.size(); i++) { |
|||
Long startTime = partitionsStartTime.get(i); |
|||
Long endTime; |
|||
if (partitionsStartTime.size() - 1 == i) { |
|||
endTime = System.currentTimeMillis(); |
|||
} else { |
|||
endTime = partitionsStartTime.get(i + 1) - 1; |
|||
} |
|||
partitions.put(startTime, endTime); |
|||
} |
|||
return partitions; |
|||
} |
|||
|
|||
public Result<ObjectType> getResult(UUID tenantId) { |
|||
return resultStore.getStoredResult(tenantId); |
|||
} |
|||
|
|||
public void cancelExport(UUID tenantId) { |
|||
boolean removed = storage.cleanUpExportData(tenantId); |
|||
if (!removed) { |
|||
throw new IllegalArgumentException("Not found"); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public ResponseEntity<InputStreamResource> downloadResult(UUID tenantId) { |
|||
var result = resultStore.getStoredResult(tenantId); |
|||
if (result == null || !result.isDone()) { |
|||
throw new IllegalStateException("Not ready yet"); |
|||
} else if (!result.isSuccess()) { |
|||
throw new IllegalStateException("Tenant export failed: " + result.getError()); |
|||
} |
|||
|
|||
String fileName = "tenant_export_data_" + tenantId + ".tar"; |
|||
return ResponseEntity.ok() |
|||
.header("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE) |
|||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + fileName) |
|||
.header("x-filename", fileName) |
|||
.body(new InputStreamResource(storage.downloadExportData(tenantId))); |
|||
} |
|||
|
|||
|
|||
private void exportTelemetry(TenantId tenantId, TenantExportConfig exportConfig) { |
|||
storage.readAndProcess(LATEST_TS_KV, tenantId.getId(), dataWrapper -> { |
|||
LatestTsKv latestTsKv = (LatestTsKv) dataWrapper.getEntity(); |
|||
EntityId entityId = latestTsKv.getEntityId(); |
|||
String key = latestTsKv.getEntry().getKey(); |
|||
|
|||
// todo: submit for 10 latest kv, wait
|
|||
try { |
|||
timeseriesDao.findAllAsync(tenantId, entityId, key, tsKvEntry -> { |
|||
TsKv tsKv = new TsKv(entityId, tsKvEntry, exportConfig.getTsKvTtl()); |
|||
save(tenantId, TS_KV, tsKv); |
|||
}).get(60, TimeUnit.MINUTES); |
|||
} catch (Exception e) { |
|||
throw new RuntimeException(e); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private void exportAuditLogs(TenantId tenantId, TenantExportConfig exportConfig) { |
|||
Map<Long, Long> partitions = getPartitions(ModelConstants.AUDIT_LOG_TABLE_NAME); |
|||
partitions.forEach((startTime, endTime) -> { |
|||
PageDataIterable<AuditLog> auditLogs = new PageDataIterable<>(pageLink -> { |
|||
return auditLogDao.findAuditLogsByTenantId(tenantId.getId(), null, new TimePageLink(pageLink, startTime, endTime)); |
|||
}, 512); |
|||
for (AuditLog auditLog : auditLogs) { |
|||
save(tenantId, AUDIT_LOG, auditLog); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private void exportAttributes(TenantId tenantId, EntityId entityId) { |
|||
for (AttributeScope attributeScope : AttributeScope.values()) { |
|||
List<AttributeKvEntry> attributes = attributesDao.findAll(tenantId, entityId, attributeScope); |
|||
for (AttributeKvEntry entry : attributes) { |
|||
AttributeKv attributeKv = new AttributeKv(entityId, attributeScope, entry); |
|||
save(tenantId, ATTRIBUTE_KV, attributeKv); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void exportRelations(TenantId tenantId, EntityId entityId) { |
|||
List<EntityRelation> relations = relationDao.findAllByFrom(tenantId, entityId); |
|||
for (EntityRelation relation : relations) { |
|||
save(tenantId, RELATION, relation); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void exportLatestTelemetry(TenantId tenantId, EntityId entityId) { |
|||
List<TsKvEntry> latestTelemetry = timeseriesLatestDao.findAllLatest(tenantId, entityId).get(30, TimeUnit.SECONDS); |
|||
for (TsKvEntry tsKvEntry : latestTelemetry) { |
|||
LatestTsKv latestTsKv = new LatestTsKv(entityId, tsKvEntry); |
|||
save(tenantId, LATEST_TS_KV, latestTsKv); |
|||
} |
|||
} |
|||
|
|||
private void exportEvents(TenantId tenantId, EntityId entityId) { |
|||
for (EventType eventType : EventType.values()) { |
|||
Map<Long, Long> partitions = getPartitions(eventType.getTable()); |
|||
partitions.forEach((startTime, endTime) -> { |
|||
PageDataIterable<? extends Event> events = new PageDataIterable<>(pageLink -> { |
|||
return eventDao.findEvents(tenantId.getId(), entityId.getId(), eventType, new TimePageLink(pageLink, startTime, endTime)); |
|||
}, 512); |
|||
for (Event event : events) { |
|||
save(tenantId, EVENT, event); |
|||
} |
|||
}); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,144 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant; |
|||
|
|||
import jakarta.annotation.PostConstruct; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.exception.ExceptionUtils; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.cache.CacheManager; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.transaction.support.TransactionTemplate; |
|||
import org.thingsboard.common.util.ThingsBoardThreadFactory; |
|||
import org.thingsboard.server.common.data.CacheConstants; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.dao.entity.EntityDaoRegistry; |
|||
import org.thingsboard.server.dao.tenant.TenantProfileService; |
|||
import org.thingsboard.server.service.sync.tenant.util.DataWrapper; |
|||
import org.thingsboard.server.service.sync.tenant.util.Result; |
|||
import org.thingsboard.server.service.sync.tenant.util.ResultStore; |
|||
import org.thingsboard.server.service.sync.tenant.util.Storage; |
|||
import org.thingsboard.server.service.sync.tenant.util.TenantImportConfig; |
|||
|
|||
import java.io.InputStream; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ExecutorService; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.atomic.AtomicReference; |
|||
|
|||
import static org.thingsboard.server.common.data.ObjectType.TENANT; |
|||
|
|||
@Service |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class TenantImportService { |
|||
|
|||
private final Storage storage; |
|||
private final EntityDaoRegistry entityDaoRegistry; |
|||
private final TenantProfileService tenantProfileService; |
|||
private final TransactionTemplate transactionTemplate; |
|||
private final CacheManager cacheManager; |
|||
private ResultStore<ObjectType> resultStore; |
|||
@Value("${cache.specs.tenantImportResults.timeToLiveInMinutes:1440}") |
|||
private int resultsTtl; |
|||
|
|||
private final ExecutorService executor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tenant-import")); |
|||
|
|||
@PostConstruct |
|||
private void init() { |
|||
resultStore = ResultStore.<ObjectType>builder() |
|||
.name("Tenant import") |
|||
.ttlInMinutes(resultsTtl) |
|||
.persistFrequency(100) |
|||
.cacheName(CacheConstants.TENANT_IMPORT_RESULT_CACHE) |
|||
.cacheManager(cacheManager) |
|||
.build(); |
|||
} |
|||
|
|||
public UUID importTenant(InputStream dataStream, TenantImportConfig config) { |
|||
storage.unwrapImportData(dataStream); |
|||
|
|||
AtomicReference<Tenant> tenant = new AtomicReference<>(); |
|||
storage.readAndProcess(TENANT, dataWrapper -> { |
|||
tenant.set((Tenant) dataWrapper.getEntity()); |
|||
}); |
|||
if (tenant.get() == null) { |
|||
throw new IllegalStateException("No tenant data found"); |
|||
} |
|||
UUID tenantId = tenant.get().getUuidId(); |
|||
|
|||
executor.submit(() -> { |
|||
try { |
|||
transactionTemplate.executeWithoutResult(status -> { |
|||
importTenant(tenant.get(), config); |
|||
}); |
|||
resultStore.update(tenantId, result -> { |
|||
result.setSuccess(true); |
|||
result.setDone(true); |
|||
}); |
|||
} catch (Throwable t) { |
|||
log.error("Failed to import tenant", t); |
|||
try { |
|||
resultStore.update(tenantId, result -> { |
|||
result.setError(ExceptionUtils.getStackTrace(t)); |
|||
result.setDone(true); |
|||
}); |
|||
} catch (Exception e) { |
|||
log.error("Failed to handle import error", e); |
|||
} |
|||
} |
|||
storage.cleanUpImportData(); |
|||
}); |
|||
|
|||
return tenantId; |
|||
} |
|||
|
|||
private void importTenant(Tenant tenant, TenantImportConfig config) { |
|||
TenantId tenantId = tenant.getId(); |
|||
for (ObjectType type : ObjectType.values()) { // TODO: in parallel for related entities + ts kv
|
|||
log.debug("[{}] Importing {} entities", tenantId, type); |
|||
storage.readAndProcess(type, dataWrapper -> { |
|||
save(tenantId, type, dataWrapper); |
|||
}); |
|||
resultStore.flush(tenantId.getId(), type); |
|||
} |
|||
|
|||
clearCaches(); |
|||
} |
|||
|
|||
private void save(TenantId tenantId, ObjectType type, DataWrapper dataWrapper) { |
|||
Object entity = dataWrapper.getEntity(); |
|||
if (entity instanceof Tenant tenant) { |
|||
tenant.setTenantProfileId(tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID).getId()); |
|||
} |
|||
|
|||
entityDaoRegistry.getDao(type).save(tenantId, entity); |
|||
resultStore.report(tenantId.getId(), type); |
|||
log.trace("[{}][{}] Imported entity {}", tenantId, type, entity); |
|||
} |
|||
|
|||
private void clearCaches() { |
|||
cacheManager.getCacheNames().forEach(cacheName -> cacheManager.getCache(cacheName).clear()); |
|||
} |
|||
|
|||
public Result<ObjectType> getResult(UUID tenantId) { |
|||
return resultStore.getStoredResult(tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,58 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
|
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
public class DataWrapper { |
|||
|
|||
@JsonIgnoreProperties(ignoreUnknown = true) |
|||
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) |
|||
private Object entity; |
|||
|
|||
private Map<String, String> additionalInfo; |
|||
|
|||
private DataWrapper(Object entity) { |
|||
this.entity = entity; |
|||
} |
|||
|
|||
public static DataWrapper of(Object entity) { |
|||
return new DataWrapper(entity); |
|||
} |
|||
|
|||
public void putAdditionalInfo(String key, String value) { |
|||
if (additionalInfo == null) { |
|||
additionalInfo = new HashMap<>(); |
|||
} |
|||
additionalInfo.put(key, value); |
|||
} |
|||
|
|||
public String getAdditionalInfo(String key) { |
|||
if (additionalInfo == null) { |
|||
return null; |
|||
} |
|||
return additionalInfo.get(key); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,43 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import lombok.Data; |
|||
|
|||
import java.io.Serializable; |
|||
import java.util.LinkedHashMap; |
|||
import java.util.Map; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.atomic.AtomicInteger; |
|||
|
|||
@Data |
|||
public class Result<K> implements Serializable { |
|||
|
|||
private boolean done; |
|||
private boolean success; |
|||
private String error; |
|||
|
|||
private final Map<K, AtomicInteger> stats = new LinkedHashMap<>(); |
|||
|
|||
public int report(K key) { |
|||
return stats.computeIfAbsent(key, k -> new AtomicInteger()).incrementAndGet(); |
|||
} |
|||
|
|||
public int getCount(K key) { |
|||
return Optional.ofNullable(stats.get(key)).map(AtomicInteger::get).orElse(0); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,103 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import com.github.benmanes.caffeine.cache.Cache; |
|||
import com.github.benmanes.caffeine.cache.Caffeine; |
|||
import lombok.Builder; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.springframework.cache.CacheManager; |
|||
|
|||
import java.util.Optional; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.function.BiConsumer; |
|||
import java.util.function.Consumer; |
|||
|
|||
@Slf4j |
|||
public class ResultStore<K> { |
|||
|
|||
private final String name; |
|||
private final int persistFrequency; |
|||
|
|||
private final Cache<UUID, Result<K>> results; |
|||
private final org.springframework.cache.Cache resultsCache; |
|||
|
|||
@Builder |
|||
public ResultStore(String name, int ttlInMinutes, int persistFrequency, BiConsumer<UUID, Result<K>> removalListener, |
|||
String cacheName, CacheManager cacheManager) { |
|||
this.name = name; |
|||
this.persistFrequency = persistFrequency; |
|||
|
|||
this.resultsCache = Optional.ofNullable(cacheManager.getCache(cacheName)) |
|||
.orElseThrow(() -> new IllegalArgumentException(cacheName + " cache is missing")); |
|||
this.results = Caffeine.newBuilder() |
|||
.expireAfterAccess(ttlInMinutes, TimeUnit.MINUTES) |
|||
.<UUID, Result<K>>removalListener((key, result, removalCause) -> { |
|||
if (key != null) { |
|||
if (removalListener != null) { |
|||
removalListener.accept(key, result); |
|||
} |
|||
resultsCache.evict(key); |
|||
} |
|||
}) |
|||
.build(); |
|||
} |
|||
|
|||
public void report(UUID id, K key) { |
|||
Result<K> result = getOrCreateResult(id); |
|||
int count = result.report(key); |
|||
if (count % persistFrequency == 0) { |
|||
resultsCache.put(id, result); |
|||
print(id, key, count); |
|||
} |
|||
} |
|||
|
|||
public int get(UUID id, K key) { |
|||
Result<K> result = getOrCreateResult(id); |
|||
return result.getCount(key); |
|||
} |
|||
|
|||
public void update(UUID id, Consumer<Result<K>> updater) { |
|||
Result<K> result = getOrCreateResult(id); |
|||
updater.accept(result); |
|||
flush(id); |
|||
} |
|||
|
|||
public void flush(UUID id, K... keys) { |
|||
Result<K> result = getOrCreateResult(id); |
|||
resultsCache.put(id, result); |
|||
for (K key : keys) { |
|||
int count = result.getCount(key); |
|||
if (count % persistFrequency != 0) { // if we didn't print it before
|
|||
print(id, key, count); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private void print(UUID id, K key, int count) { |
|||
log.info("[{}][{}][{}] {} processed", name, id, key, count); |
|||
} |
|||
|
|||
private Result<K> getOrCreateResult(UUID id) { |
|||
return results.get(id, k -> new Result<K>()); |
|||
} |
|||
|
|||
public Result<K> getStoredResult(UUID id) { |
|||
return resultsCache.get(id, Result.class); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,193 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import com.fasterxml.jackson.core.JsonProcessingException; |
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.compress.archivers.tar.TarArchiveEntry; |
|||
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; |
|||
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; |
|||
import org.apache.commons.io.FileUtils; |
|||
import org.apache.commons.io.IOUtils; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
|
|||
import java.io.BufferedReader; |
|||
import java.io.FileInputStream; |
|||
import java.io.FileOutputStream; |
|||
import java.io.IOException; |
|||
import java.io.InputStream; |
|||
import java.io.InputStreamReader; |
|||
import java.io.OutputStreamWriter; |
|||
import java.io.UncheckedIOException; |
|||
import java.io.Writer; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.util.HashMap; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.ConcurrentHashMap; |
|||
import java.util.function.Consumer; |
|||
import java.util.zip.GZIPInputStream; |
|||
import java.util.zip.GZIPOutputStream; |
|||
|
|||
@Service |
|||
@Slf4j |
|||
public class Storage { |
|||
|
|||
@Value("${tenant.export.working_directory:/tmp/tenants-export-import}") |
|||
private String workingDirectory; |
|||
|
|||
private static final ObjectMapper jsonMapper = new ObjectMapper(); |
|||
|
|||
private final Map<UUID, Map<Path, Writer>> files = new ConcurrentHashMap<>(); |
|||
|
|||
@SneakyThrows |
|||
public void init(UUID tenantId) { |
|||
Path workingDirectory = Path.of(this.workingDirectory, tenantId.toString()); |
|||
Files.createDirectories(workingDirectory); |
|||
files.put(tenantId, new HashMap<>()); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void save(UUID tenantId, ObjectType type, DataWrapper dataWrapper) { |
|||
Path file = getExportDataPath(tenantId, type); |
|||
Map<Path, Writer> dataFiles = files.get(tenantId); |
|||
if (dataFiles == null) { |
|||
throw new IllegalStateException("Export cancelled"); |
|||
} |
|||
Writer writer = dataFiles.computeIfAbsent(file, path -> { |
|||
try { |
|||
Files.deleteIfExists(file); |
|||
Files.createFile(file); |
|||
|
|||
FileOutputStream fileOutputStream = new FileOutputStream(file.toFile()); |
|||
return new OutputStreamWriter(new GZIPOutputStream(fileOutputStream), StandardCharsets.UTF_8); |
|||
} catch (IOException e) { |
|||
throw new UncheckedIOException(e); |
|||
} |
|||
}); |
|||
|
|||
String data = toJson(dataWrapper); |
|||
writer.write(data + System.lineSeparator()); |
|||
log.trace("[{}] Saved entity to {}: {}", tenantId, file, dataWrapper.getEntity()); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void archiveExportData(UUID tenantId) { |
|||
for (Writer writer : files.get(tenantId).values()) { |
|||
writer.close(); |
|||
} |
|||
|
|||
Map<Path, Writer> files = this.files.remove(tenantId); |
|||
TarArchiveOutputStream tarArchive = new TarArchiveOutputStream(new FileOutputStream(Path.of(workingDirectory, tenantId.toString(), "data.tar").toFile())); |
|||
for (Path file : files.keySet()) { |
|||
TarArchiveEntry archiveEntry = new TarArchiveEntry(file, file.getFileName().toString()); |
|||
tarArchive.putArchiveEntry(archiveEntry); |
|||
Files.copy(file, tarArchive); |
|||
tarArchive.closeArchiveEntry(); |
|||
Files.delete(file); |
|||
} |
|||
tarArchive.close(); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public boolean cleanUpExportData(UUID tenantId) { |
|||
Path workingDirectory = Path.of(this.workingDirectory, tenantId.toString()); |
|||
deleteDirectory(workingDirectory); |
|||
return files.remove(tenantId) != null; |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public InputStream downloadExportData(UUID tenantId) { |
|||
return Files.newInputStream(Path.of(workingDirectory, tenantId.toString(), "data.tar")); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void unwrapImportData(InputStream dataStream) { |
|||
Path workingDirectory = Path.of(this.workingDirectory, "importing"); |
|||
deleteDirectory(workingDirectory); |
|||
Files.createDirectories(workingDirectory); |
|||
|
|||
TarArchiveInputStream tarArchive = new TarArchiveInputStream(dataStream); |
|||
TarArchiveEntry archiveEntry; |
|||
while ((archiveEntry = tarArchive.getNextEntry()) != null) { |
|||
Path file = workingDirectory.resolve(archiveEntry.getName()); |
|||
Files.createFile(file); |
|||
IOUtils.copy(tarArchive, Files.newOutputStream(file)); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void cleanUpImportData() { |
|||
Path workingDirectory = Path.of(this.workingDirectory, "importing"); |
|||
deleteDirectory(workingDirectory); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
public void readAndProcess(ObjectType type, UUID tenantId, Consumer<DataWrapper> processor) { |
|||
Path file = getExportDataPath(tenantId, type); |
|||
Writer writer = files.get(tenantId).get(file); |
|||
if (writer != null) { |
|||
writer.close(); |
|||
} |
|||
readAndProcess(type, file, processor); |
|||
} |
|||
|
|||
public void readAndProcess(ObjectType type, Consumer<DataWrapper> processor) { |
|||
Path file = Path.of(this.workingDirectory, "importing", type.name().toLowerCase() + ".gz"); |
|||
readAndProcess(type, file, processor); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void readAndProcess(ObjectType type, Path file, Consumer<DataWrapper> processor) { |
|||
if (!Files.exists(file)) { |
|||
log.debug("No data for {}", type); |
|||
return; |
|||
} |
|||
|
|||
FileInputStream fileInputStream = new FileInputStream(file.toFile()); |
|||
try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(fileInputStream)))) { |
|||
reader.lines().forEach(line -> { |
|||
DataWrapper dataWrapper; |
|||
try { |
|||
dataWrapper = jsonMapper.readValue(line, DataWrapper.class); |
|||
} catch (JsonProcessingException e) { |
|||
throw new RuntimeException(e); |
|||
} |
|||
processor.accept(dataWrapper); |
|||
}); |
|||
} |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private String toJson(Object o) { |
|||
return jsonMapper.writeValueAsString(o); |
|||
} |
|||
|
|||
private Path getExportDataPath(UUID tenantId, ObjectType type) { |
|||
return Path.of(workingDirectory, tenantId.toString(), type.name().toLowerCase() + ".gz"); |
|||
} |
|||
|
|||
private void deleteDirectory(Path directory) throws IOException { |
|||
FileUtils.deleteDirectory(directory.toFile()); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
|
|||
import java.util.Collections; |
|||
import java.util.Set; |
|||
import java.util.UUID; |
|||
|
|||
@Data |
|||
public class TenantExportConfig { |
|||
|
|||
private UUID tenantId; |
|||
private Set<ObjectType> skipped; |
|||
|
|||
private long tsKvTtl; |
|||
|
|||
public Set<ObjectType> getSkipped() { |
|||
return skipped != null ? skipped : Collections.emptySet(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.tenant.util; |
|||
|
|||
import lombok.Data; |
|||
|
|||
@Data |
|||
public class TenantImportConfig { |
|||
|
|||
} |
|||
@ -0,0 +1,383 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.tenant; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import lombok.SneakyThrows; |
|||
import org.junit.After; |
|||
import org.junit.Before; |
|||
import org.junit.Test; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.http.HttpMethod; |
|||
import org.springframework.jdbc.core.JdbcTemplate; |
|||
import org.springframework.mock.web.MockMultipartFile; |
|||
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; |
|||
import org.thingsboard.server.common.data.AdminSettings; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.Customer; |
|||
import org.thingsboard.server.common.data.Dashboard; |
|||
import org.thingsboard.server.common.data.DeviceProfile; |
|||
import org.thingsboard.server.common.data.EventInfo; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.OtaPackage; |
|||
import org.thingsboard.server.common.data.Tenant; |
|||
import org.thingsboard.server.common.data.audit.ActionType; |
|||
import org.thingsboard.server.common.data.audit.AuditLog; |
|||
import org.thingsboard.server.common.data.event.EventType; |
|||
import org.thingsboard.server.common.data.event.LifecycleEvent; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.HasId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.id.UUIDBased; |
|||
import org.thingsboard.server.common.data.kv.AttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; |
|||
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; |
|||
import org.thingsboard.server.common.data.kv.BasicTsKvEntry; |
|||
import org.thingsboard.server.common.data.kv.TsKvEntry; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.common.data.page.TimePageLink; |
|||
import org.thingsboard.server.common.data.queue.Queue; |
|||
import org.thingsboard.server.common.data.relation.EntityRelation; |
|||
import org.thingsboard.server.common.data.rpc.Rpc; |
|||
import org.thingsboard.server.common.data.rule.RuleChain; |
|||
import org.thingsboard.server.common.data.rule.RuleNode; |
|||
import org.thingsboard.server.controller.AbstractControllerTest; |
|||
import org.thingsboard.server.dao.TenantEntityDao; |
|||
import org.thingsboard.server.dao.attributes.AttributesService; |
|||
import org.thingsboard.server.dao.audit.AuditLogDao; |
|||
import org.thingsboard.server.dao.entity.EntityDaoRegistry; |
|||
import org.thingsboard.server.dao.event.EventService; |
|||
import org.thingsboard.server.dao.housekeeper.CleanUpService; |
|||
import org.thingsboard.server.dao.relation.RelationService; |
|||
import org.thingsboard.server.dao.service.DaoSqlTest; |
|||
import org.thingsboard.server.dao.tenant.TenantDao; |
|||
import org.thingsboard.server.dao.timeseries.TimeseriesService; |
|||
import org.thingsboard.server.service.sync.tenant.util.Result; |
|||
import org.thingsboard.server.service.sync.tenant.util.TenantExportConfig; |
|||
import org.thingsboard.server.utils.TestUtils; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Arrays; |
|||
import java.util.Collections; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
import static org.assertj.core.api.Assertions.fail; |
|||
import static org.awaitility.Awaitility.await; |
|||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; |
|||
import static org.thingsboard.server.common.data.ObjectType.ADMIN_SETTINGS; |
|||
import static org.thingsboard.server.common.data.ObjectType.ATTRIBUTE_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.AUDIT_LOG; |
|||
import static org.thingsboard.server.common.data.ObjectType.CUSTOMER; |
|||
import static org.thingsboard.server.common.data.ObjectType.DASHBOARD; |
|||
import static org.thingsboard.server.common.data.ObjectType.DEVICE_PROFILE; |
|||
import static org.thingsboard.server.common.data.ObjectType.EVENT; |
|||
import static org.thingsboard.server.common.data.ObjectType.LATEST_TS_KV; |
|||
import static org.thingsboard.server.common.data.ObjectType.OTA_PACKAGE; |
|||
import static org.thingsboard.server.common.data.ObjectType.QUEUE; |
|||
import static org.thingsboard.server.common.data.ObjectType.RPC; |
|||
import static org.thingsboard.server.common.data.ObjectType.RULE_CHAIN; |
|||
import static org.thingsboard.server.common.data.ObjectType.RULE_NODE; |
|||
import static org.thingsboard.server.common.data.ObjectType.TENANT; |
|||
import static org.thingsboard.server.common.data.ObjectType.TS_KV; |
|||
import static org.thingsboard.server.utils.TestUtils.newRandomizedEntity; |
|||
|
|||
@DaoSqlTest |
|||
@SuppressWarnings("unchecked") |
|||
public class TenantExportImportTest extends AbstractControllerTest { |
|||
|
|||
@Autowired |
|||
private EventService eventService; |
|||
@Autowired |
|||
private AuditLogDao auditLogDao; |
|||
@Autowired |
|||
private TimeseriesService timeseriesService; |
|||
@Autowired |
|||
private AttributesService attributesService; |
|||
@Autowired |
|||
private RelationService relationService; |
|||
@Autowired |
|||
private EntityDaoRegistry entityDaoRegistry; |
|||
@Autowired |
|||
private TenantDao tenantDao; |
|||
@Autowired |
|||
private CleanUpService cleanUpService; |
|||
@Autowired |
|||
private JdbcTemplate jdbcTemplate; |
|||
|
|||
private TenantId tenantId; |
|||
private Map<ObjectType, List<HasId>> createdEntities; |
|||
private Map<ObjectType, Map<EntityId, Object>> createdRelatedEntities; |
|||
|
|||
@Before |
|||
public void before() throws Exception { |
|||
createdEntities = new HashMap<>(); |
|||
createdRelatedEntities = new HashMap<>(); |
|||
loginSysAdmin(); |
|||
} |
|||
|
|||
@After |
|||
public void after() { |
|||
|
|||
} |
|||
|
|||
@Test |
|||
public void testExportImport() throws Exception { |
|||
Tenant tenant = newRandomizedEntity(Tenant.class); |
|||
tenant.setTenantProfileId(tenantProfileId); |
|||
this.tenantId = tenant.getId(); |
|||
saveEntity(tenant, TENANT); |
|||
fillTenant(); |
|||
|
|||
Result<ObjectType> exportResult = exportTenant(tenantId); |
|||
createdEntities.forEach((type, entities) -> { |
|||
int exportedCount = exportResult.getCount(type); |
|||
if (exportedCount != entities.size()) { |
|||
fail("expected " + entities.size() + " exported entities of type " + type + " but was " + exportedCount + ". " + |
|||
"expected: " + entities + ", actual: " + findAllOfType(type)); |
|||
return; |
|||
} |
|||
assertAllPresent(type, entities); |
|||
}); |
|||
byte[] exportData = doGet("/api/tenant/export/result/" + tenantId + "/download").andExpect(status().isOk()).andReturn().getResponse().getContentAsByteArray(); |
|||
|
|||
loginSysAdmin(); |
|||
createdEntities.values().forEach(entities -> { |
|||
for (HasId entity : entities) { |
|||
if (entity.getId() instanceof EntityId entityId) { |
|||
cleanUpService.cleanUpRelatedData(tenantId, entityId); |
|||
} |
|||
} |
|||
}); |
|||
jdbcTemplate.execute("TRUNCATE TABLE audit_log"); |
|||
deleteTenant(tenantId); |
|||
await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { |
|||
createdEntities.forEach((type, entities) -> { |
|||
assertThat(findAllOfType(type)).isEmpty(); |
|||
entities.forEach(entity -> { |
|||
if (entity.getId() instanceof EntityId entityId) { |
|||
assertThat(findAuditLogs(entityId)).isEmpty(); |
|||
assertThat(findEvents(entityId)).isEmpty(); |
|||
assertThat(findAttributes(entityId)).isEmpty(); |
|||
assertThat(findLatestTs(entityId)).isEmpty(); |
|||
TsKvEntry latestKv = (TsKvEntry) createdRelatedEntities.get(LATEST_TS_KV).get(entityId); |
|||
assertThat(findTsHistory(entityId, latestKv.getKey())).isEmpty(); |
|||
} |
|||
}); |
|||
}); |
|||
}); |
|||
|
|||
Result<ObjectType> importResult = importTenant(exportData); |
|||
createdEntities.forEach((type, entities) -> { |
|||
int importedCount = importResult.getCount(type); |
|||
if (importedCount != entities.size()) { |
|||
fail("expected " + entities.size() + " imported entities of type " + type + " but was " + importedCount + ". " + |
|||
"expected: " + entities + ", actual: " + findAllOfType(type)); |
|||
return; |
|||
} |
|||
assertAllPresent(type, entities); |
|||
}); |
|||
} |
|||
|
|||
private void assertAllPresent(ObjectType type, List<HasId> expectedEntities) { |
|||
assertThat(findAllOfType(type)).containsAll(expectedEntities); |
|||
expectedEntities.forEach(entity -> { |
|||
if (entity.getId() instanceof EntityId entityId) { |
|||
assertThat((List) findAttributes(entityId)).containsExactly(createdRelatedEntities.get(ATTRIBUTE_KV).get(entityId)); |
|||
List<TsKvEntry> latestTs = findLatestTs(entityId); |
|||
assertThat((List) latestTs).containsExactly(createdRelatedEntities.get(LATEST_TS_KV).get(entityId)); |
|||
assertThat((List) findTsHistory(entityId, latestTs.get(0).getKey())).containsExactly(createdRelatedEntities.get(TS_KV).get(entityId)); |
|||
assertThat((List) findAuditLogs(entityId)).containsExactly(createdRelatedEntities.get(AUDIT_LOG).get(entityId)); |
|||
assertThat((List) findEvents(entityId)).containsExactly(createdRelatedEntities.get(EVENT).get(entityId)); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
private Result<ObjectType> exportTenant(TenantId tenantId) { |
|||
TenantExportConfig exportConfig = new TenantExportConfig(); |
|||
exportConfig.setTenantId(tenantId.getId()); |
|||
doPost("/api/tenant/export", exportConfig, UUID.class); |
|||
Result<ObjectType> result = await().atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.until(() -> doGetTyped("/api/tenant/export/result/" + tenantId.getId(), new TypeReference<>() {}), Result::isDone); |
|||
System.err.println(result); |
|||
if (!result.isSuccess()) { |
|||
throw new RuntimeException(result.getError()); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private Result<ObjectType> importTenant(byte[] data) throws Exception { |
|||
MockMultipartFile dataFile = new MockMultipartFile("dataFile", "data.tar", "application/octet-stream", data); |
|||
var request = MockMvcRequestBuilders.multipart(HttpMethod.POST, "/api/tenant/import").file(dataFile); |
|||
setJwtToken(request); |
|||
UUID tenantId = readResponse(mockMvc.perform(request).andExpect(status().isOk()), UUID.class); |
|||
|
|||
Result<ObjectType> result = await().atMost(TIMEOUT, TimeUnit.SECONDS) |
|||
.until(() -> doGetTyped("/api/tenant/import/result/" + tenantId, new TypeReference<>() {}), Result::isDone); |
|||
System.err.println(result); |
|||
if (!result.isSuccess()) { |
|||
throw new RuntimeException(result.getError()); |
|||
} |
|||
return result; |
|||
} |
|||
|
|||
private void fillTenant() { |
|||
Customer customer = newRandomizedEntity(Customer.class); |
|||
customer.setTenantId(tenantId); |
|||
saveEntity(customer, CUSTOMER); |
|||
|
|||
AdminSettings adminSettings = newRandomizedEntity(AdminSettings.class); |
|||
adminSettings.setTenantId(tenantId); |
|||
saveEntity(adminSettings, ADMIN_SETTINGS); |
|||
|
|||
Queue queue = newRandomizedEntity(Queue.class); |
|||
queue.setTenantId(tenantId); |
|||
saveEntity(queue, QUEUE); |
|||
|
|||
Rpc rpc = newRandomizedEntity(Rpc.class); |
|||
rpc.setTenantId(tenantId); |
|||
saveEntity(rpc, RPC); |
|||
|
|||
RuleChain ruleChain = newRandomizedEntity(RuleChain.class); |
|||
ruleChain.setTenantId(tenantId); |
|||
saveEntity(ruleChain, RULE_CHAIN); |
|||
|
|||
for (int i = 0; i < 10; i++) { |
|||
RuleNode ruleNode = newRandomizedEntity(RuleNode.class); |
|||
ruleNode.setRuleChainId(ruleChain.getId()); |
|||
saveEntity(ruleNode, RULE_NODE); |
|||
createRelation(ruleNode.getId(), ruleChain.getId()); |
|||
createRelation(ruleChain.getId(), ruleNode.getId()); |
|||
} |
|||
|
|||
Dashboard dashboard = newRandomizedEntity(Dashboard.class); |
|||
dashboard.setTenantId(tenantId); |
|||
saveEntity(dashboard, DASHBOARD); |
|||
|
|||
DeviceProfile deviceProfile = newRandomizedEntity(DeviceProfile.class); |
|||
|
|||
OtaPackage otaPackage = newRandomizedEntity(OtaPackage.class); |
|||
otaPackage.setTenantId(tenantId); |
|||
otaPackage.setDeviceProfileId(deviceProfile.getId()); |
|||
saveEntity(otaPackage, OTA_PACKAGE); |
|||
|
|||
deviceProfile.setTenantId(tenantId); |
|||
deviceProfile.setFirmwareId(otaPackage.getId()); |
|||
deviceProfile.setSoftwareId(otaPackage.getId()); |
|||
deviceProfile.setDefaultEdgeRuleChainId(null); |
|||
deviceProfile.setDefaultRuleChainId(ruleChain.getId()); |
|||
deviceProfile.setDefaultDashboardId(dashboard.getId()); |
|||
saveEntity(deviceProfile, DEVICE_PROFILE); |
|||
} |
|||
|
|||
private <T extends HasId<? extends UUIDBased>> T saveEntity(T entity, ObjectType type) { |
|||
entity = (T) entityDaoRegistry.getDao(type).save(tenantId, entity); |
|||
if (entity.getId() instanceof EntityId entityId) { |
|||
createAttribute(entityId); |
|||
createTelemetry(entityId); |
|||
createAuditLog(entityId); |
|||
createEvent(entityId); |
|||
} |
|||
createdEntities.computeIfAbsent(type, k -> new ArrayList<>()).add(entity); |
|||
return entity; |
|||
} |
|||
|
|||
private List<Object> findAllOfType(ObjectType type) { |
|||
if (type == TENANT) { |
|||
Tenant tenant = tenantDao.findById(TenantId.SYS_TENANT_ID, tenantId.getId()); |
|||
if (tenant == null) { |
|||
return Collections.emptyList(); |
|||
} |
|||
return List.of(tenant); |
|||
} |
|||
TenantEntityDao<?> dao = entityDaoRegistry.getTenantEntityDao(type); |
|||
PageData stored = dao.findAllByTenantId(tenantId, new PageLink(1000)); |
|||
return stored.getData(); |
|||
} |
|||
|
|||
private void createRelation(EntityId to, EntityId from) { |
|||
EntityRelation relation = TestUtils.newRandomizedEntity(EntityRelation.class); |
|||
relation.setTo(to); |
|||
relation.setFrom(from); |
|||
relationService.saveRelation(tenantId, relation); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void createAttribute(EntityId entityId) { |
|||
BaseAttributeKvEntry attributeKv = newRandomizedEntity(BaseAttributeKvEntry.class); |
|||
attributesService.save(tenantId, entityId, AttributeScope.SHARED_SCOPE, attributeKv).get(); |
|||
createdRelatedEntities.computeIfAbsent(ATTRIBUTE_KV, k -> new HashMap<>()).put(entityId, attributeKv); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private List<AttributeKvEntry> findAttributes(EntityId entityId) { |
|||
return attributesService.findAll(tenantId, entityId, AttributeScope.SHARED_SCOPE).get(); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void createTelemetry(EntityId entityId) { |
|||
BasicTsKvEntry tsKvEntry = newRandomizedEntity(BasicTsKvEntry.class); |
|||
timeseriesService.save(tenantId, entityId, tsKvEntry).get(); |
|||
createdRelatedEntities.computeIfAbsent(TS_KV, k -> new HashMap<>()).put(entityId, tsKvEntry); |
|||
createdRelatedEntities.computeIfAbsent(LATEST_TS_KV, k -> new HashMap<>()).put(entityId, tsKvEntry); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private List<TsKvEntry> findLatestTs(EntityId entityId) { |
|||
return timeseriesService.findAllLatest(tenantId, entityId).get(); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private List<TsKvEntry> findTsHistory(EntityId entityId, String key) { |
|||
return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(key, 0, System.currentTimeMillis(), 100, "ASC"))).get(); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void createEvent(EntityId entityId) { |
|||
LifecycleEvent event = LifecycleEvent.builder() |
|||
.tenantId(tenantId) |
|||
.entityId(entityId.getId()) |
|||
.serviceId("test") |
|||
.lcEventType("test") |
|||
.success(true) |
|||
.build(); |
|||
eventService.saveAsync(event).get(); |
|||
createdRelatedEntities.computeIfAbsent(EVENT, k -> new HashMap<>()).put(entityId, findEvents(entityId).get(0)); |
|||
} |
|||
|
|||
private List<EventInfo> findEvents(EntityId entityId) { |
|||
return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100, 0)).getData(); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private void createAuditLog(EntityId entityId) { |
|||
AuditLog auditLog = newRandomizedEntity(AuditLog.class); |
|||
auditLog.setTenantId(tenantId); |
|||
auditLog.setEntityId(entityId); |
|||
auditLog = auditLogDao.save(tenantId, auditLog); |
|||
createdRelatedEntities.computeIfAbsent(AUDIT_LOG, k -> new HashMap<>()).put(entityId, auditLog); |
|||
} |
|||
|
|||
private List<AuditLog> findAuditLogs(EntityId entityId) { |
|||
return auditLogDao.findAuditLogsByTenantIdAndEntityId(tenantId.getId(), entityId, Arrays.stream(ActionType.values()).toList(), new TimePageLink(100, 0)).getData(); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,75 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.utils; |
|||
|
|||
import com.fasterxml.jackson.databind.JsonNode; |
|||
import org.apache.commons.lang3.RandomStringUtils; |
|||
import org.apache.commons.lang3.RandomUtils; |
|||
import org.jeasy.random.EasyRandom; |
|||
import org.jeasy.random.EasyRandomParameters; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; |
|||
import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; |
|||
import org.thingsboard.server.common.data.device.data.DeviceConfiguration; |
|||
import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.AllowCreateNewDevicesDeviceProfileProvisionConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; |
|||
import org.thingsboard.server.common.data.device.profile.DeviceProfileConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.DeviceProfileProvisionConfiguration; |
|||
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; |
|||
import org.thingsboard.server.common.data.id.DeviceId; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.kv.DoubleDataEntry; |
|||
import org.thingsboard.server.common.data.kv.KvEntry; |
|||
import org.thingsboard.server.common.data.kv.StringDataEntry; |
|||
|
|||
import java.nio.ByteBuffer; |
|||
import java.util.UUID; |
|||
|
|||
public class TestUtils { |
|||
|
|||
static EasyRandom easyRandom; |
|||
|
|||
static { |
|||
EasyRandomParameters parameters = new EasyRandomParameters() |
|||
.randomize(DeviceConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceConfiguration.class)) |
|||
.randomize(DeviceTransportConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceTransportConfiguration.class)) |
|||
.randomize(DeviceProfileConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceProfileConfiguration.class)) |
|||
.randomize(DeviceProfileTransportConfiguration.class, () -> easyRandom.nextObject(DefaultDeviceProfileTransportConfiguration.class)) |
|||
.randomize(DeviceProfileProvisionConfiguration.class, () -> easyRandom.nextObject(AllowCreateNewDevicesDeviceProfileProvisionConfiguration.class)) |
|||
.randomize(DeviceProfileAlarm.class, DeviceProfileAlarm::new) |
|||
.randomize(JsonNode.class, () -> JacksonUtil.newObjectNode() |
|||
.put(RandomStringUtils.randomAlphanumeric(10), RandomStringUtils.randomAlphanumeric(10)) |
|||
.put(RandomStringUtils.randomAlphanumeric(10), easyRandom.nextDouble())) |
|||
.randomize(EntityId.class, () -> new DeviceId(UUID.randomUUID())) |
|||
.randomize(KvEntry.class, () -> easyRandom.nextBoolean() ? easyRandom.nextObject(StringDataEntry.class) : easyRandom.nextObject(DoubleDataEntry.class)) |
|||
.randomize(Long.class, () -> RandomUtils.nextLong(1, Integer.MAX_VALUE)) |
|||
.randomize(ByteBuffer.class, () -> { |
|||
byte[] bytes = new byte[50]; |
|||
easyRandom.nextBytes(bytes); |
|||
return ByteBuffer.wrap(bytes); |
|||
}); |
|||
|
|||
easyRandom = new EasyRandom(parameters); |
|||
} |
|||
|
|||
public static <T> T newRandomizedEntity(Class<T> type) { |
|||
return easyRandom.nextObject(type); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,27 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data.alarm; |
|||
|
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
@Data |
|||
public class AlarmType { |
|||
|
|||
private TenantId tenantId; |
|||
private String type; |
|||
|
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data.kv; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.server.common.data.AttributeScope; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
public class AttributeKv { |
|||
private EntityId entityId; |
|||
private AttributeScope scope; |
|||
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) |
|||
@JsonIgnoreProperties(ignoreUnknown = true) |
|||
private AttributeKvEntry entry; |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data.kv; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
public class LatestTsKv { |
|||
private EntityId entityId; |
|||
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) |
|||
@JsonIgnoreProperties(ignoreUnknown = true) |
|||
private TsKvEntry entry; |
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.common.data.kv; |
|||
|
|||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; |
|||
import com.fasterxml.jackson.annotation.JsonTypeInfo; |
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
|
|||
@Data |
|||
@AllArgsConstructor |
|||
@NoArgsConstructor |
|||
public class TsKv { |
|||
private EntityId entityId; |
|||
@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) |
|||
@JsonIgnoreProperties(ignoreUnknown = true) |
|||
private TsKvEntry entry; |
|||
private long ttl; |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao; |
|||
|
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
public interface ObjectDao<T> { // TODO: choose a better name
|
|||
|
|||
T save(TenantId tenantId, T entity); |
|||
|
|||
default ObjectType getType() { |
|||
return null; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.model.sql; |
|||
|
|||
import lombok.AllArgsConstructor; |
|||
import lombok.Data; |
|||
import lombok.NoArgsConstructor; |
|||
|
|||
import java.io.Serializable; |
|||
import java.util.UUID; |
|||
|
|||
@Data |
|||
@NoArgsConstructor |
|||
@AllArgsConstructor |
|||
public class AlarmTypeCompositeKey implements Serializable { |
|||
|
|||
private UUID tenantId; |
|||
private String type; |
|||
|
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.model.sql; |
|||
|
|||
import jakarta.persistence.Column; |
|||
import jakarta.persistence.Entity; |
|||
import jakarta.persistence.Id; |
|||
import jakarta.persistence.IdClass; |
|||
import jakarta.persistence.Table; |
|||
import lombok.Data; |
|||
import org.thingsboard.server.common.data.alarm.AlarmType; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.dao.model.ModelConstants; |
|||
import org.thingsboard.server.dao.model.ToData; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
@Data |
|||
@Entity |
|||
@Table(name = ModelConstants.ALARM_TYPES_TABLE_NAME) |
|||
@IdClass(AlarmTypeCompositeKey.class) |
|||
public class AlarmTypeEntity implements ToData<AlarmType> { |
|||
|
|||
@Id |
|||
@Column(name = ModelConstants.TENANT_ID_PROPERTY, nullable = false) |
|||
private UUID tenantId; |
|||
|
|||
@Id |
|||
@Column(name = ModelConstants.ALARM_TYPE_PROPERTY, nullable = false) |
|||
private String type; |
|||
|
|||
public AlarmTypeEntity() {} |
|||
|
|||
public AlarmTypeEntity(AlarmType alarmType) { |
|||
setTenantId(alarmType.getTenantId().getId()); |
|||
setType(alarmType.getType()); |
|||
} |
|||
|
|||
public AlarmTypeEntity(UUID tenantId, String type) { |
|||
this.tenantId = tenantId; |
|||
this.type = type; |
|||
} |
|||
|
|||
@Override |
|||
public AlarmType toData() { |
|||
AlarmType alarmType = new AlarmType(); |
|||
alarmType.setTenantId(TenantId.fromUUID(tenantId)); |
|||
alarmType.setType(type); |
|||
return alarmType; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.sql.alarm; |
|||
|
|||
import org.springframework.data.domain.Page; |
|||
import org.springframework.data.domain.Pageable; |
|||
import org.springframework.data.jpa.repository.JpaRepository; |
|||
import org.springframework.data.jpa.repository.Modifying; |
|||
import org.springframework.data.jpa.repository.Query; |
|||
import org.springframework.data.repository.query.Param; |
|||
import org.springframework.transaction.annotation.Transactional; |
|||
import org.thingsboard.server.dao.model.sql.AlarmTypeCompositeKey; |
|||
import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; |
|||
import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; |
|||
import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; |
|||
|
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
public interface AlarmTypeRepository extends JpaRepository<AlarmTypeEntity, AlarmTypeCompositeKey> { |
|||
|
|||
Page<AlarmTypeEntity> findByTenantId(UUID tenantId, Pageable pageable); |
|||
|
|||
} |
|||
@ -0,0 +1,52 @@ |
|||
/** |
|||
* Copyright © 2016-2024 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.sql.alarm; |
|||
|
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.common.data.ObjectType; |
|||
import org.thingsboard.server.common.data.alarm.AlarmType; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.page.PageLink; |
|||
import org.thingsboard.server.dao.DaoUtil; |
|||
import org.thingsboard.server.dao.TenantEntityDao; |
|||
import org.thingsboard.server.dao.model.sql.AlarmTypeEntity; |
|||
import org.thingsboard.server.dao.util.SqlDao; |
|||
|
|||
@Component |
|||
@SqlDao |
|||
public class JpaAlarmTypeDao implements TenantEntityDao<AlarmType> { |
|||
|
|||
@Autowired |
|||
private AlarmTypeRepository alarmTypeRepository; |
|||
|
|||
@Override |
|||
public PageData<AlarmType> findAllByTenantId(TenantId tenantId, PageLink pageLink) { |
|||
return DaoUtil.toPageData(alarmTypeRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink, "tenantId", "type"))); |
|||
} |
|||
|
|||
@Override |
|||
public AlarmType save(TenantId tenantId, AlarmType alarmType) { |
|||
return alarmTypeRepository.save(new AlarmTypeEntity(alarmType)).toData(); |
|||
} |
|||
|
|||
@Override |
|||
public ObjectType getType() { |
|||
return ObjectType.ALARM_TYPE; |
|||
} |
|||
|
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue