Merge branch 'feature/bulk-import/device-credentials' of https://github.com/vvlladd28/thingsboard into feature/bulk-import/device-credentials

ui-ngx/src/app/core/http/asset.service.ts

@@ -22,6 +22,7 @@ import { PageLink } from '@shared/models/page/page-link';
 import { PageData } from '@shared/models/page/page-data';
 import { EntitySubtype } from '@app/shared/models/entity-type.models';
 import { Asset, AssetInfo, AssetSearchQuery } from '@app/shared/models/asset.models';
+import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
 
 @Injectable({
   providedIn: 'root'
@@ -105,4 +106,8 @@ export class AssetService {
       defaultHttpOptionsFromConfig(config));
   }
 
+  public bulkImportAssets(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
+    return this.http.post<BulkImportResult>('/api/asset/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
+  }
+
 }

ui-ngx/src/app/core/http/device.service.ts

@@ -30,6 +30,7 @@ import {
 } from '@app/shared/models/device.models';
 import { EntitySubtype } from '@app/shared/models/entity-type.models';
 import { AuthService } from '@core/auth/auth.service';
+import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
 
 @Injectable({
   providedIn: 'root'
@@ -170,7 +171,11 @@ export class DeviceService {
   public getEdgeDevices(edgeId: string, pageLink: PageLink, type: string = '',
                         config?: RequestConfig): Observable<PageData<DeviceInfo>> {
     return this.http.get<PageData<DeviceInfo>>(`/api/edge/${edgeId}/devices${pageLink.toQuery()}&type=${type}`,
-      defaultHttpOptionsFromConfig(config))
+      defaultHttpOptionsFromConfig(config));
+  }
+
+  public bulkImportDevices(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
+    return this.http.post<BulkImportResult>('/api/device/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
   }
 
 }

ui-ngx/src/app/core/http/edge.service.ts

@@ -23,6 +23,7 @@ import { PageData } from '@shared/models/page/page-data';
 import { EntitySubtype } from '@app/shared/models/entity-type.models';
 import { Edge, EdgeEvent, EdgeInfo, EdgeSearchQuery } from '@shared/models/edge.models';
 import { EntityId } from '@shared/models/id/entity-id';
+import { BulkImportRequest, BulkImportResult } from '@home/components/import-export/import-export.models';
 
 @Injectable({
   providedIn: 'root'
@@ -59,7 +60,7 @@ export class EdgeService {
   }
 
   public getCustomerEdgeInfos(customerId: string, pageLink: PageLink, type: string = '',
-                               config?: RequestConfig): Observable<PageData<EdgeInfo>> {
+                              config?: RequestConfig): Observable<PageData<EdgeInfo>> {
     return this.http.get<PageData<EdgeInfo>>(`/api/customer/${customerId}/edgeInfos${pageLink.toQuery()}&type=${type}`,
       defaultHttpOptionsFromConfig(config));
   }
@@ -108,4 +109,8 @@ export class EdgeService {
   public findByName(edgeName: string, config?: RequestConfig): Observable<Edge> {
     return this.http.get<Edge>(`/api/tenant/edges?edgeName=${edgeName}`, defaultHttpOptionsFromConfig(config));
   }
+
+  public bulkImportEdges(entitiesData: BulkImportRequest, config?: RequestConfig): Observable<BulkImportResult> {
+    return this.http.post<BulkImportResult>('/api/edge/bulk_import', entitiesData, defaultHttpOptionsFromConfig(config));
+  }
 }

ui-ngx/src/app/modules/home/components/import-export/import-dialog-csv.component.html

@@ -113,23 +113,23 @@
       </mat-step>
       <mat-step>
         <ng-template matStepLabel>{{ 'import.stepper-text.creat-entities' | translate }}</ng-template>
-        <mat-progress-bar color="warn" class="tb-import-progress" mode="determinate" [value]="progressCreate">
+        <mat-progress-bar color="warn" class="tb-import-progress" mode="indeterminate">
         </mat-progress-bar>
       </mat-step>
       <mat-step>
         <ng-template matStepLabel>{{ 'import.stepper-text.done' | translate }}</ng-template>
         <div fxLayout="column">
-          <p class="mat-body-1" *ngIf="this.statistical?.create && this.statistical?.create.entity">
+          <p class="mat-body-1" *ngIf="this.statistical?.created">
-            {{ translate.instant('import.message.create-entities', {count: this.statistical.create.entity}) }}
+            {{ translate.instant('import.message.create-entities', {count: this.statistical.created}) }}
           </p>
-          <p class="mat-body-1" *ngIf="this.statistical?.update && this.statistical?.update.entity">
+          <p class="mat-body-1" *ngIf="this.statistical?.updated">
-            {{ translate.instant('import.message.update-entities', {count: this.statistical.update.entity}) }}
+            {{ translate.instant('import.message.update-entities', {count: this.statistical.updated}) }}
           </p>
-          <p class="mat-body-1" style="margin-bottom: 0.8em" *ngIf="this.statistical?.error && this.statistical?.error.entity">
+          <p class="mat-body-1" style="margin-bottom: 0.8em" *ngIf="this.statistical?.errors">
-            {{ translate.instant('import.message.error-entities', {count: this.statistical.error.entity}) }}
+            {{ translate.instant('import.message.error-entities', {count: this.statistical.errors}) }}
           </p>
           <mat-expansion-panel class="advanced-logs" [expanded]="false"
-                               *ngIf="this.statistical?.error && this.statistical?.error.entity"
+                               *ngIf="this.statistical?.errorsList?.length"
                                (opened)="initEditor()">
             <mat-expansion-panel-header [collapsedHeight]="'38px'" [expandedHeight]="'38px'">
               <mat-panel-title>

ui-ngx/src/app/modules/home/components/import-export/import-dialog-csv.component.ts

@@ -26,18 +26,18 @@ import { TranslateService } from '@ngx-translate/core';
 import { ActionNotificationShow } from '@core/notification/notification.actions';
 import { MatVerticalStepper } from '@angular/material/stepper';
 import {
+  BulkImportRequest,
+  BulkImportResult,
+  ColumnMapping,
   convertCSVToJson,
   CsvColumnParam,
+  CSVDelimiter,
   CsvToJsonConfig,
   CsvToJsonResult,
   ImportEntityColumnType
 } from '@home/components/import-export/import-export.models';
-import { EdgeImportEntityData, ImportEntitiesResultInfo, ImportEntityData } from '@app/shared/models/entity.models';
 import { ImportExportService } from '@home/components/import-export/import-export.service';
 import { TableColumnsAssignmentComponent } from '@home/components/import-export/table-columns-assignment.component';
-import { getDeviceCredentialMQTTDefault } from '@shared/models/device.models';
-import { isDefinedAndNotNull } from '@core/utils';
-import { getLwm2mSecurityConfigModelsDefault } from '@shared/models/lwm2m-security-config.models';
 import { Ace } from 'ace-builds';
 import { getAce } from '@shared/models/ace/ace.models';
 
@@ -68,7 +68,7 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
   importTitle: string;
   importFileLabel: string;
 
-  delimiters: { key: string, value: string }[] = [{
+  delimiters: { key: CSVDelimiter, value: string }[] = [{
     key: ',',
     value: ','
   }, {
@@ -89,8 +89,7 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
   columnTypesFormGroup: FormGroup;
 
   isImportData = false;
-  progressCreate = 0;
+  statistical: BulkImportResult;
-  statistical: ImportEntitiesResultInfo;
 
   private allowAssignColumn: ImportEntityColumnType[];
   private initEditorComponent = false;
@@ -215,169 +214,16 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
 
 
   private addEntities() {
-    const importData = this.parseData;
+    const entitiesData: BulkImportRequest = {
-    const isHeader: boolean = this.importParametersFormGroup.get('isHeader').value;
+      file: this.selectFileFormGroup.get('importData').value,
-    const parameterColumns: CsvColumnParam[] = this.columnTypesFormGroup.get('columnsParam').value;
+      mapping: {
-    const entitiesData: ImportEntityData[] = [];
+        columns: this.processingColumnsParams(),
-    let sentDataLength = 0;
+        delimiter: this.importParametersFormGroup.get('delim').value,
-    const startLineNumber = isHeader ? 2 : 1;
+        header: this.importParametersFormGroup.get('isHeader').value,
-    for (let row = 0; row < importData.rows.length; row++) {
+        update: this.importParametersFormGroup.get('isUpdate').value
-      const entityData: ImportEntityData = this.constructDraftImportEntityData();
-      const i = row;
-      entityData.lineNumber = startLineNumber + i;
-      for (let j = 0; j < parameterColumns.length; j++) {
-        if (!isDefinedAndNotNull(importData.rows[i][j]) || importData.rows[i][j] === '') {
-          continue;
-        }
-        switch (parameterColumns[j].type) {
-          case ImportEntityColumnType.serverAttribute:
-            entityData.attributes.server.push({
-              key: parameterColumns[j].key,
-              value: importData.rows[i][j]
-            });
-            break;
-          case ImportEntityColumnType.timeseries:
-            entityData.timeseries.push({
-              key: parameterColumns[j].key,
-              value: importData.rows[i][j]
-            });
-            break;
-          case ImportEntityColumnType.sharedAttribute:
-            entityData.attributes.shared.push({
-              key: parameterColumns[j].key,
-              value: importData.rows[i][j]
-            });
-            break;
-          case ImportEntityColumnType.name:
-            entityData.name = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.type:
-            entityData.type = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.label:
-            entityData.label = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.isGateway:
-            entityData.gateway = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.description:
-            entityData.description = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.accessToken:
-            entityData.credential.accessToken = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.x509:
-            entityData.credential.x509 = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.mqttClientId:
-            if (!entityData.credential.mqtt) {
-              entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
-            }
-            entityData.credential.mqtt.clientId = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.mqttUserName:
-            if (!entityData.credential.mqtt) {
-              entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
-            }
-            entityData.credential.mqtt.userName = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.mqttPassword:
-            if (!entityData.credential.mqtt) {
-              entityData.credential.mqtt = getDeviceCredentialMQTTDefault();
-            }
-            entityData.credential.mqtt.password = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mClientEndpoint:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.client.endpoint = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mClientSecurityConfigMode:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.client.securityConfigClientMode = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mClientIdentity:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.client.identity = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mClientKey:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.client.key = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mClientCert:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.client.cert = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mBootstrapServerSecurityMode:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.bootstrapServer.securityMode = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mBootstrapServerClientPublicKeyOrId:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.bootstrapServer.clientPublicKeyOrId = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mBootstrapServerClientSecretKey:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.bootstrapServer.clientSecretKey = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mServerSecurityMode:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.lwm2mServer.securityMode = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mServerClientPublicKeyOrId:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.lwm2mServer.clientPublicKeyOrId = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.lwm2mServerClientSecretKey:
-            if (!entityData.credential.lwm2m) {
-              entityData.credential.lwm2m = getLwm2mSecurityConfigModelsDefault();
-            }
-            entityData.credential.lwm2m.bootstrap.lwm2mServer.clientSecretKey = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.edgeLicenseKey:
-            (entityData as EdgeImportEntityData).edgeLicenseKey = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.cloudEndpoint:
-            (entityData as EdgeImportEntityData).cloudEndpoint = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.routingKey:
-            (entityData as EdgeImportEntityData).routingKey = importData.rows[i][j];
-            break;
-          case ImportEntityColumnType.secret:
-            (entityData as EdgeImportEntityData).secret = importData.rows[i][j];
-            break;
-        }
       }
-      entitiesData.push(entityData);
-    }
-    const createImportEntityCompleted = () => {
-      sentDataLength++;
-      this.progressCreate = Math.round((sentDataLength / importData.rows.length) * 100);
     };
-
+    this.importExport.bulkImportEntities(entitiesData, this.entityType, {ignoreErrors: true}).subscribe(
-    const isUpdate: boolean = this.importParametersFormGroup.get('isUpdate').value;
-
-    this.importExport.importEntities(entitiesData, this.entityType, isUpdate,
-      createImportEntityCompleted, {ignoreErrors: true, resendRequest: true}).subscribe(
       (result) => {
         this.statistical = result;
         this.isImportData = false;
@@ -386,36 +232,22 @@ export class ImportDialogCsvComponent extends DialogComponent<ImportDialogCsvCom
     );
   }
 
-  private constructDraftImportEntityData(): ImportEntityData {
+  private processingColumnsParams(): Array<ColumnMapping> {
-    const entityData: ImportEntityData = {
+    const parameterColumns: CsvColumnParam[] = this.columnTypesFormGroup.get('columnsParam').value;
-      lineNumber: 1,
+    const allowKeyForTypeColumns: ImportEntityColumnType[] = [
-      name: '',
+      ImportEntityColumnType.serverAttribute,
-      type: '',
+      ImportEntityColumnType.timeseries,
-      description: '',
+      ImportEntityColumnType.sharedAttribute
-      gateway: null,
+    ];
-      label: '',
+    return parameterColumns.map(column => ({
-      attributes: {
+      type: column.type,
-        server: [],
+      key: allowKeyForTypeColumns.some(type => type === column.type) ? column.key : undefined
-        shared: []
+    }));
-      },
-      credential: {},
-      timeseries: []
-    };
-    if (this.entityType === EntityType.EDGE) {
-      const edgeEntityData: EdgeImportEntityData = entityData as EdgeImportEntityData;
-      edgeEntityData.edgeLicenseKey = '';
-      edgeEntityData.cloudEndpoint = '';
-      edgeEntityData.routingKey = '';
-      edgeEntityData.secret = '';
-      return edgeEntityData;
-    } else {
-      return entityData;
-    }
   }
 
   initEditor() {
     if (!this.initEditorComponent) {
-      this.createEditor(this.failureDetailsEditorElmRef, this.statistical.error.errors);
+      this.createEditor(this.failureDetailsEditorElmRef, this.statistical.errorsList.join('\n'));
     }
   }
 

ui-ngx/src/app/modules/home/components/import-export/import-export.models.ts

@@ -38,6 +38,8 @@ export interface CsvToJsonResult {
   rows?: any[][];
 }
 
+export type CSVDelimiter = ',' | ';' | '|' | '\t';
+
 export enum ImportEntityColumnType {
   name = 'NAME',
   type = 'TYPE',
@@ -113,6 +115,28 @@ export interface CsvColumnParam {
   sampleData: any;
 }
 
+export interface ColumnMapping {
+  type: ImportEntityColumnType;
+  key?: string;
+}
+
+export interface BulkImportRequest {
+  file: string;
+  mapping: {
+    columns: Array<ColumnMapping>;
+    delimiter: CSVDelimiter;
+    header: boolean;
+    update: boolean;
+  };
+}
+
+export interface BulkImportResult {
+  created: number;
+  updated: number;
+  errors: number;
+  errorsList: Array<string>;
+}
+
 export interface FileType {
   mimeType: string;
   extension: string;

ui-ngx/src/app/modules/home/components/import-export/import-export.service.ts

@@ -44,7 +44,15 @@ import {
   EntityAliasesDialogData
 } from '@home/components/alias/entity-aliases-dialog.component';
 import { ItemBufferService, WidgetItem } from '@core/services/item-buffer.service';
-import { FileType, ImportWidgetResult, JSON_TYPE, WidgetsBundleItem, ZIP_TYPE } from './import-export.models';
+import {
+  BulkImportRequest,
+  BulkImportResult,
+  FileType,
+  ImportWidgetResult,
+  JSON_TYPE,
+  WidgetsBundleItem,
+  ZIP_TYPE
+} from './import-export.models';
 import { AliasEntityType, EntityType } from '@shared/models/entity-type.models';
 import { UtilsService } from '@core/services/utils.service';
 import { WidgetService } from '@core/http/widget.service';
@@ -59,6 +67,9 @@ import { DeviceProfileService } from '@core/http/device-profile.service';
 import { DeviceProfile } from '@shared/models/device.models';
 import { TenantProfile } from '@shared/models/tenant.model';
 import { TenantProfileService } from '@core/http/tenant-profile.service';
+import { DeviceService } from '@core/http/device.service';
+import { AssetService } from '@core/http/asset.service';
+import { EdgeService } from '@core/http/edge.service';
 
 // @dynamic
 @Injectable()
@@ -75,6 +86,9 @@ export class ImportExportService {
               private tenantProfileService: TenantProfileService,
               private entityService: EntityService,
               private ruleChainService: RuleChainService,
+              private deviceService: DeviceService,
+              private assetService: AssetService,
+              private edgeService: EdgeService,
               private utils: UtilsService,
               private itembuffer: ItemBufferService,
               private dialog: MatDialog) {
@@ -342,6 +356,17 @@ export class ImportExportService {
     );
   }
 
+  public bulkImportEntities(entitiesData: BulkImportRequest, entityType: EntityType, config?: RequestConfig): Observable<BulkImportResult> {
+    switch (entityType) {
+      case EntityType.DEVICE:
+        return this.deviceService.bulkImportDevices(entitiesData, config);
+      case EntityType.ASSET:
+        return this.assetService.bulkImportAssets(entitiesData, config);
+      case EntityType.EDGE:
+        return this.edgeService.bulkImportEdges(entitiesData, config);
+    }
+  }
+
   public importEntities(entitiesData: ImportEntityData[], entityType: EntityType, updateData: boolean,
                         importEntityCompleted?: () => void, config?: RequestConfig): Observable<ImportEntitiesResultInfo> {
     let partSize = 100;