From eb50685108bc3b893ee90604cda34eeb39588bfd Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 15 Mar 2024 11:28:04 +0200 Subject: [PATCH 1/3] Version control - not rollback on error option --- .../DefaultEntitiesExportImportService.java | 6 +++- .../DefaultEntitiesVersionControlService.java | 33 +++++++++++-------- .../sync/vc/data/EntitiesImportCtx.java | 1 + .../load/EntityTypeVersionLoadRequest.java | 1 + .../entity-types-version-load.component.html | 5 +++ .../assets/locale/locale.constant-en_US.json | 4 ++- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 4e3d034207..08a3cc2303 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -101,7 +101,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); ctx.addReferenceCallback(exportData.getExternalId(), importResult.getSaveReferencesCallback()); - ctx.addEventCallback(importResult.getSendEventsCallback()); + if (ctx.isRollbackOnError()) { + ctx.addEventCallback(importResult.getSendEventsCallback()); + } else { + importResult.getSendEventsCallback().run(); + } return importResult; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 5c0dcc930d..123bb8a191 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -246,16 +246,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + ctx.setRollbackOnError(true); VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); DonAsynchron.withCallback(future, - entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)), + entityData -> load(ctx, request, c -> loadSingleEntity(c, config, entityData)), e -> processLoadError(ctx, e), executor); break; } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - executor.submit(() -> doInTemplate(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest))); + ctx.setRollbackOnError(versionLoadRequest.isRollbackOnError()); + executor.submit(() -> load(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest))); break; } default: @@ -265,19 +267,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return ctx.getRequestId(); } - private VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function function) { + private VersionLoadResult load(EntitiesImportCtx ctx, VersionLoadRequest request, Function loadFunction) { try { - VersionLoadResult result = transactionTemplate.execute(status -> { - try { - return function.apply(ctx); - } catch (RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); // to prevent UndeclaredThrowableException + VersionLoadResult result; + if (ctx.isRollbackOnError()) { + result = transactionTemplate.execute(status -> { + try { + return loadFunction.apply(ctx); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); // to prevent UndeclaredThrowableException + } + }); + for (ThrowingRunnable eventCallback : ctx.getEventCallbacks()) { + eventCallback.run(); } - }); - for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { - throwingRunnable.run(); + } else { + result = loadFunction.apply(ctx); } result.setDone(true); return cachePut(ctx.getRequestId(), result); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 0f0e3313b8..19999ea080 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -58,6 +58,7 @@ public class EntitiesImportCtx { private boolean finalImportAttempt = false; private EntityImportSettings settings; private EntityImportResult currentImportResult; + private boolean rollbackOnError; public EntitiesImportCtx(UUID requestId, User user, String versionId) { this(requestId, user, versionId, null); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java index bcc6fc9c5b..7eca2c04de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -26,6 +26,7 @@ import java.util.Map; public class EntityTypeVersionLoadRequest extends VersionLoadRequest { private Map entityTypes; + private boolean rollbackOnError; @Override public VersionLoadRequestType getType() { diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index 620d406132..0deb31e3f4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -99,3 +99,8 @@ +
+ + {{ 'version-control.rollback-on-error' | translate }} + +
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 8f02129a80..571773f389 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4915,7 +4915,9 @@ "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}.", "runtime-failed": "Failed: {{message}}", - "auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings." + "auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings.", + "rollback-on-error": "Rollback on error", + "rollback-on-error-hint": "If you have a large amount of entities to restore, consider disabling this option to increase performance.
Note, if an error occurs over the course of version loading, already persisted entities (with relations, attributes, etc.) will stay as is" }, "widget": { "widget-library": "Widgets library", From 2ca3a86c96f1e948d2ff28d6b0025be606791514 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 15 Mar 2024 16:51:00 +0200 Subject: [PATCH 2/3] UI: Add in VC support new property rollback on error --- .../home/components/vc/complex-version-load.component.html | 7 ++++++- .../home/components/vc/complex-version-load.component.ts | 1 + .../components/vc/entity-types-version-load.component.html | 5 ----- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html index 0ef7f3bc7c..ccab83b9aa 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -23,11 +23,16 @@ -
+
+
+ + {{ 'version-control.rollback-on-error' | translate }} + +
-
- - {{ 'version-control.rollback-on-error' | translate }} - -
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 571773f389..3fc05736e8 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -4917,7 +4917,7 @@ "runtime-failed": "Failed: {{message}}", "auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings.", "rollback-on-error": "Rollback on error", - "rollback-on-error-hint": "If you have a large amount of entities to restore, consider disabling this option to increase performance.
Note, if an error occurs over the course of version loading, already persisted entities (with relations, attributes, etc.) will stay as is" + "rollback-on-error-hint": "If you have a large amount of entities to restore, consider disabling this option to increase performance.\n Note, if an error occurs over the course of version loading, already persisted entities (with relations, attributes, etc.) will stay as is" }, "widget": { "widget-library": "Widgets library", From a4384eb0191e91e14ab73c6183a3d6af8886fcd8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 15 Mar 2024 18:27:47 +0200 Subject: [PATCH 3/3] VC: fix other entities removal --- .../DefaultEntitiesVersionControlService.java | 45 +++++++++++++------ .../vc/complex-version-load.component.ts | 3 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 123bb8a191..c4454bff9a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -89,6 +90,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.function.Function; @@ -331,7 +333,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont sw.startNew("Entities " + entityType.name()); ctx.setSettings(getEntityImportSettings(request, entityType)); importEntities(ctx, entityType); - persistToCache(ctx); } sw.startNew("Reimport"); @@ -343,7 +344,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) .forEach(entityType -> removeOtherEntities(ctx, entityType)); - persistToCache(ctx); sw.startNew("References and Relations"); exportImportService.saveReferencesAndRelations(ctx); @@ -396,6 +396,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } + + persistToCache(ctx); log.debug("Imported {} pack ({}) for tenant {}", entityType, entityDataList.size(), ctx.getTenantId()); offset += limit; } while (entityDataList.size() == limit); @@ -420,19 +422,34 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { - exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); - - ctx.addEventCallback(() -> { - entityNotificationService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null, - ActionType.DELETED, ctx.getUser()); - }); - ctx.registerDeleted(entityType); + var entities = new PageDataIterable<>(link -> exportableEntitiesService.findEntitiesIdsByTenantId(ctx.getTenantId(), entityType, link), 100); + Set toRemove = new HashSet<>(); + for (EntityId entityId : entities) { + if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entityId)) { + toRemove.add(entityId); } - }); + } + + for (EntityId entityId : toRemove) { + ExportableEntity entity = exportableEntitiesService.findEntityById(entityId); + exportableEntitiesService.removeById(ctx.getTenantId(), entityId); + + ThrowingRunnable callback = () -> { + entityNotificationService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null, + ActionType.DELETED, ctx.getUser()); + }; + if (ctx.isRollbackOnError()) { + ctx.addEventCallback(callback); + } else { + try { + callback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + ctx.registerDeleted(entityType); + } + persistToCache(ctx); } private VersionLoadResult onError(EntityId externalId, Throwable e) { diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts index 77a3ca3194..0fed8dab88 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -79,7 +79,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit ngOnInit(): void { this.loadVersionFormGroup = this.fb.group({ entityTypes: [createDefaultEntityTypesVersionLoad(), []], - rollbackOnError: [false] + rollbackOnError: [true] }); } @@ -117,6 +117,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit const request: EntityTypeVersionLoadRequest = { versionId: this.versionId, entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + rollbackOnError: this.loadVersionFormGroup.get('rollbackOnError').value, type: VersionLoadRequestType.ENTITY_TYPE }; this.versionLoadResult$ = this.entitiesVersionControlService.loadEntitiesVersion(request, {ignoreErrors: true}).pipe( diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 8cf7366a0a..ba54a297a7 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -144,6 +144,7 @@ export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; type: VersionLoadRequestType.ENTITY_TYPE; + rollbackOnError: boolean; } export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} {