Merge pull request #10377 from thingsboard/fix/vc-performance

VC restore: optional rollback on error to vastly increase performance

application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@@ -101,7 +101,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
         ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
 
         ctx.addReferenceCallback(exportData.getExternalId(), importResult.getSaveReferencesCallback());
-        ctx.addEventCallback(importResult.getSendEventsCallback());
+        if (ctx.isRollbackOnError()) {
+            ctx.addEventCallback(importResult.getSendEventsCallback());
+        } else {
+            importResult.getSendEventsCallback().run();
+        }
         return importResult;
     }
 

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
 import org.thingsboard.server.common.data.id.HasId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.page.PageData;
+import org.thingsboard.server.common.data.page.PageDataIterable;
 import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
@@ -89,6 +90,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
@@ -246,16 +248,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         switch (request.getType()) {
             case SINGLE_ENTITY: {
                 SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
+                ctx.setRollbackOnError(true);
                 VersionLoadConfig config = versionLoadRequest.getConfig();
                 ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
                 DonAsynchron.withCallback(future,
-                        entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)),
+                        entityData -> load(ctx, request, c -> loadSingleEntity(c, config, entityData)),
                         e -> processLoadError(ctx, e), executor);
                 break;
             }
             case ENTITY_TYPE: {
                 EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request;
-                executor.submit(() -> doInTemplate(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest)));
+                ctx.setRollbackOnError(versionLoadRequest.isRollbackOnError());
+                executor.submit(() -> load(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest)));
                 break;
             }
             default:
@@ -265,19 +269,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         return ctx.getRequestId();
     }
 
-    private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
+    private <R> VersionLoadResult load(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> loadFunction) {
         try {
-            VersionLoadResult result = transactionTemplate.execute(status -> {
+            VersionLoadResult result;
-                try {
+            if (ctx.isRollbackOnError()) {
-                    return function.apply(ctx);
+                result = transactionTemplate.execute(status -> {
-                } catch (RuntimeException e) {
+                    try {
-                    throw e;
+                        return loadFunction.apply(ctx);
-                } catch (Exception e) {
+                    } catch (RuntimeException e) {
-                    throw new RuntimeException(e); // to prevent UndeclaredThrowableException
+                        throw e;
+                    } catch (Exception e) {
+                        throw new RuntimeException(e); // to prevent UndeclaredThrowableException
+                    }
+                });
+                for (ThrowingRunnable eventCallback : ctx.getEventCallbacks()) {
+                    eventCallback.run();
                 }
-            });
+            } else {
-            for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
+                result = loadFunction.apply(ctx);
-                throwingRunnable.run();
             }
             result.setDone(true);
             return cachePut(ctx.getRequestId(), result);
@@ -324,7 +333,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
             sw.startNew("Entities " + entityType.name());
             ctx.setSettings(getEntityImportSettings(request, entityType));
             importEntities(ctx, entityType);
-            persistToCache(ctx);
         }
 
         sw.startNew("Reimport");
@@ -336,7 +344,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
                 .sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
                 .forEach(entityType -> removeOtherEntities(ctx, entityType));
-        persistToCache(ctx);
 
         sw.startNew("References and Relations");
         exportImportService.saveReferencesAndRelations(ctx);
@@ -389,6 +396,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                 ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>())
                         .add(importResult.getSavedEntity().getId());
             }
+
+            persistToCache(ctx);
             log.debug("Imported {} pack ({}) for tenant {}", entityType, entityDataList.size(), ctx.getTenantId());
             offset += limit;
         } while (entityDataList.size() == limit);
@@ -413,17 +422,34 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
     }
 
     private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) {
-        DaoUtil.processInBatches(pageLink -> {
+        var entities = new PageDataIterable<>(link -> exportableEntitiesService.findEntitiesIdsByTenantId(ctx.getTenantId(), entityType, link), 100);
-            return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink);
+        Set<EntityId> toRemove = new HashSet<>();
-        }, 100, entity -> {
+        for (EntityId entityId : entities) {
-            if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) {
+            if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entityId)) {
-                exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId());
+                toRemove.add(entityId);
-
-                ctx.addEventCallback(() -> logEntityActionService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null,
-                        ActionType.DELETED, ctx.getUser()));
-                ctx.registerDeleted(entityType);
             }
-        });
+        }
+
+        for (EntityId entityId : toRemove) {
+            ExportableEntity<EntityId> entity = exportableEntitiesService.findEntityById(entityId);
+            exportableEntitiesService.removeById(ctx.getTenantId(), entityId);
+
+            ThrowingRunnable callback = () -> {
+                logEntityActionService.logEntityAction(ctx.getTenantId(), entity.getId(), entity, null,
+                        ActionType.DELETED, ctx.getUser());
+            };
+            if (ctx.isRollbackOnError()) {
+                ctx.addEventCallback(callback);
+            } else {
+                try {
+                    callback.run();
+                } catch (ThingsboardException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+            ctx.registerDeleted(entityType);
+        }
+        persistToCache(ctx);
     }
 
     private VersionLoadResult onError(EntityId externalId, Throwable e) {

application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java

@@ -58,6 +58,7 @@ public class EntitiesImportCtx {
     private boolean finalImportAttempt = false;
     private EntityImportSettings settings;
     private EntityImportResult<?> currentImportResult;
+    private boolean rollbackOnError;
 
     public EntitiesImportCtx(UUID requestId, User user, String versionId) {
         this(requestId, user, versionId, null);

common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java

@@ -26,6 +26,7 @@ import java.util.Map;
 public class EntityTypeVersionLoadRequest extends VersionLoadRequest {
 
     private Map<EntityType, EntityTypeVersionLoadConfig> entityTypes;
+    private boolean rollbackOnError;
 
     @Override
     public VersionLoadRequestType getType() {

ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html

@@ -23,11 +23,16 @@
   <mat-progress-bar color="warn" style="z-index: 10; width: 100%; margin-bottom: -4px;" mode="indeterminate"
                     *ngIf="isLoading$ | async">
   </mat-progress-bar>
-  <form [formGroup]="loadVersionFormGroup" fxLayout="column" style="flex: 1; padding-top: 16px; overflow: auto;">
+  <form [formGroup]="loadVersionFormGroup" fxLayout="column" style="flex: 1; padding: 16px 0; overflow-y: auto; overflow-x: hidden">
     <tb-entity-types-version-load
       formControlName="entityTypes">
     </tb-entity-types-version-load>
   </form>
+  <div tb-hint-tooltip-icon="{{ 'version-control.rollback-on-error-hint' | translate }}" [formGroup]="loadVersionFormGroup">
+    <mat-slide-toggle formControlName="rollbackOnError">
+      {{ 'version-control.rollback-on-error' | translate }}
+    </mat-slide-toggle>
+  </div>
   <div fxLayoutAlign="end center" fxLayoutGap="8px" style="padding-top: 16px;">
     <button mat-button color="primary"
             type="button"

ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts

@@ -79,6 +79,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit
   ngOnInit(): void {
     this.loadVersionFormGroup = this.fb.group({
       entityTypes: [createDefaultEntityTypesVersionLoad(), []],
+      rollbackOnError: [true]
     });
   }
 
@@ -116,6 +117,7 @@ export class ComplexVersionLoadComponent extends PageComponent implements OnInit
     const request: EntityTypeVersionLoadRequest = {
       versionId: this.versionId,
       entityTypes: this.loadVersionFormGroup.get('entityTypes').value,
+      rollbackOnError: this.loadVersionFormGroup.get('rollbackOnError').value,
       type: VersionLoadRequestType.ENTITY_TYPE
     };
     this.versionLoadResult$ = this.entitiesVersionControlService.loadEntitiesVersion(request, {ignoreErrors: true}).pipe(

ui-ngx/src/app/shared/models/vc.models.ts

@@ -144,6 +144,7 @@ export interface EntityTypeVersionLoadConfig extends VersionLoadConfig {
 export interface EntityTypeVersionLoadRequest extends VersionLoadRequest {
   entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig};
   type: VersionLoadRequestType.ENTITY_TYPE;
+  rollbackOnError: boolean;
 }
 
 export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} {

ui-ngx/src/assets/locale/locale.constant-en_US.json

@@ -5016,7 +5016,9 @@
         "device-credentials-conflict": "Failed to load the device with external id <b>{{entityId}}</b><br/>due to the same credentials are already present in the database for another device.<br/>Please consider disabling the <b>load credentials</b> setting in the restore form.",
         "missing-referenced-entity": "Failed to load the <b>{{sourceEntityTypeName}}</b> with external id <b>{{sourceEntityId}}</b><br/>because it references missing <b>{{targetEntityTypeName}}</b> with id <b>{{targetEntityId}}</b>.",
         "runtime-failed": "<b>Failed:</b> {{message}}",
-        "auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings."
+        "auto-commit-settings-read-only-hint": "Auto-commit feature doesn't work with enabled read-only option in Repository settings.",
+        "rollback-on-error": "Rollback on error",
+        "rollback-on-error-hint": "If you have a large amount of entities to restore, consider disabling this option to increase performance.\n Note, if an error occurs over the course of version loading, already persisted entities (with relations, attributes, etc.) will stay as is"
     },
     "widget": {
         "widget-library": "Widgets library",