tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Refactor version create request; fixes for git api usage

pull/6559/head
Viacheslav Klimov 4 years ago
parent
0905aff7a0
commit
9b209e4075
7 changed files with 86 additions and 129 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 45
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  2. 96
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  3. 36
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java
  4. 4
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java
  5. 3
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java
  6. 1
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java
  7. 30
      application/src/main/java/org/thingsboard/server/utils/GitRepository.java

45
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.controller;
import io.swagger.annotations.ApiModelProperty;
import io.swagger.annotations.ApiOperation;
import lombok.Data;
import lombok.RequiredArgsConstructor;
@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController {
" \"saveRelations\": true\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_LIST:" + NEW_LINE +
"COMPLEX:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_LIST\",\n" +
" \"type\": \"COMPLEX\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
"\n" +
" \"entitiesIds\": [\n" +
" {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" {\n" +
" \"entityType\": \"DEVICE_PROFILE\",\n" +
" \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" +
" }\n" +
" ],\n" +
" \"config\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"MERGE\"\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_TYPE:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_TYPE\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
" \"versionName\": \"Devices and profiles: release 2\",\n" +
" \"branch\": \"master\",\n" +
"\n" +
" \"syncStrategy\": \"OVERWRITE\",\n" +
" \"entityTypes\": {\n" +
" \"DEVICE\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"MERGE\"\n" +
" \"syncStrategy\": null,\n" +
" \"allEntities\": true,\n" +
" \"saveRelations\": true\n" +
" },\n" +
" \"DEVICE_PROFILE\": {\n" +
" \"saveRelations\": true,\n" +
" \"syncStrategy\": \"OVERWRITE\"\n" +
" \"syncStrategy\": \"MERGE\",\n" +
" \"allEntities\": false,\n" +
" \"entityIds\": [\n" +
" \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" ],\n" +
" \"saveRelations\": true\n" +
" }\n" +
" }\n" +
"}\n```")

96
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion;
import org.thingsboard.server.service.sync.vc.data.VersionCreationResult;
import org.thingsboard.server.service.sync.vc.data.VersionLoadResult;
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy;
@ -88,6 +90,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
repository.fetch();
if (repository.listBranches().contains(request.getBranch())) {
repository.checkout(request.getBranch());
repository.checkout("origin/" + request.getBranch(), false);
try {
repository.checkout(request.getBranch(), true);
} catch (RefAlreadyExistsException e) {
repository.checkout(request.getBranch(), false);
}
repository.merge(request.getBranch());
} else { // TODO [viacheslav]: rollback orphan branch on failure
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
try {
repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
switch (request.getType()) {
@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig());
break;
}
case ENTITY_LIST: {
EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request;
if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) {
versionCreateRequest.getEntitiesIds().stream()
.map(EntityId::getEntityType).distinct()
.forEach(entityType -> {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
for (EntityId entityId : versionCreateRequest.getEntitiesIds()) {
saveEntityData(user, repository, entityId, versionCreateRequest.getConfig());
}
break;
}
case COMPLEX: {
ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request;
versionCreateRequest.getEntityTypes().forEach((entityType, config) -> {
if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) {
if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) {
try {
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile());
} catch (IOException e) {
@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
}
EntityTypeFilter entityTypeFilter = new EntityTypeFilter();
entityTypeFilter.setEntityType(entityType);
EntityDataPageLink entityDataPageLink = new EntityDataPageLink();
entityDataPageLink.setPage(-1);
entityDataPageLink.setPageSize(-1);
EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME);
entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC));
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList());
DaoUtil.processInBatches(pageLink -> {
entityDataPageLink.setPage(pageLink.getPage());
entityDataPageLink.setPageSize(pageLink.getPageSize());
return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query);
}, 100, data -> {
EntityId entityId = data.getEntityId();
try {
saveEntityData(user, repository, entityId, config);
} catch (Exception e) {
throw new RuntimeException(e);
if (config.isAllEntities()) {
EntityTypeFilter entityTypeFilter = new EntityTypeFilter();
entityTypeFilter.setEntityType(entityType);
EntityDataPageLink entityDataPageLink = new EntityDataPageLink();
entityDataPageLink.setPage(-1);
entityDataPageLink.setPageSize(-1);
EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME);
entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC));
EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList());
DaoUtil.processInBatches(pageLink -> {
entityDataPageLink.setPage(pageLink.getPage());
entityDataPageLink.setPageSize(pageLink.getPageSize());
return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query);
}, 100, data -> {
EntityId entityId = data.getEntityId();
try {
saveEntityData(user, repository, entityId, config);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
} else {
for (UUID entityId : config.getEntityIds()) {
try {
saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
});
}
});
break;
}
@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception {
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString());
GitRepository repository;
FileUtils.forceDelete(repositoryDirectory.toFile());
if (Files.exists(repositoryDirectory)) {
FileUtils.forceDelete(repositoryDirectory.toFile());
}
Files.createDirectories(repositoryDirectory);
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile());
@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings)))
)).get();
clearRepository(tenantId);
initRepository(tenantId, settings);
}

36
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java
View File

@ -1,36 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
@Data
@EqualsAndHashCode(callSuper = true)
public class EntityListVersionCreateRequest extends VersionCreateRequest {
private List<EntityId> entitiesIds;
private EntityTypeVersionCreateConfig config;
@Override
public VersionCreateRequestType getType() {
return VersionCreateRequestType.ENTITY_LIST;
}
}

4
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java
View File

@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.EntityId;
import java.util.List;
import java.util.UUID;
@Data
@EqualsAndHashCode(callSuper = true)
@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig {
//optional
private SyncStrategy syncStrategy;
private List<EntityId> entityIds;
private List<UUID> entityIds;
private boolean allEntities;
}

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java
View File

@ -23,8 +23,7 @@ import lombok.Data;
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
@Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class),
@Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class),
@Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class)
@Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class)
})
@Data
public abstract class VersionCreateRequest {

1
application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java
View File

@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create;
public enum VersionCreateRequestType {
SINGLE_ENTITY,
ENTITY_LIST,
COMPLEX
}

30
application/src/main/java/org/thingsboard/server/utils/GitRepository.java
View File

@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.RmCommand;
import org.eclipse.jgit.api.Status;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
public class GitRepository {
@ -84,9 +81,15 @@ public class GitRepository {
.setRemoveDeletedRefs(true));
}
public void checkout(String branch) throws GitAPIException {
public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout()
.setName("origin/" + branch));
.setCreateBranch(createBranch)
.setName(branch));
}
public void reset() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
}
public void merge(String branch) throws IOException, GitAPIException {
@ -170,14 +173,15 @@ public class GitRepository {
public void createAndCheckoutOrphanBranch(String name) throws GitAPIException {
execute(git.checkout()
.setOrphan(true)
.setForced(true)
.setName(name));
Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
if (!uncommittedChanges.isEmpty()) {
RmCommand rm = git.rm();
uncommittedChanges.forEach(rm::addFilepattern);
execute(rm);
}
execute(git.clean());
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
}
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]

Write Preview
Loading…
Cancel
Save