Andrii Shvaika
4 years ago
46 changed files with 1579 additions and 498 deletions
@ -0,0 +1,33 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
|||
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
public class CommitGitRequest extends PendingGitRequest<VersionCreationResult> { |
|||
|
|||
private final VersionCreateRequest request; |
|||
|
|||
public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { |
|||
super(tenantId); |
|||
this.request = request; |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,287 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.fasterxml.jackson.databind.ObjectWriter; |
|||
import com.fasterxml.jackson.databind.SerializationFeature; |
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import com.google.common.util.concurrent.SettableFuture; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.cluster.TbClusterService; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ExportableEntity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
|||
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
|||
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; |
|||
import org.thingsboard.server.queue.TbQueueCallback; |
|||
import org.thingsboard.server.queue.TbQueueMsgMetadata; |
|||
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
|
|||
import java.io.IOException; |
|||
import java.util.HashMap; |
|||
import java.util.List; |
|||
import java.util.Map; |
|||
import java.util.UUID; |
|||
import java.util.function.Function; |
|||
|
|||
@TbCoreComponent |
|||
@Service |
|||
@RequiredArgsConstructor |
|||
public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { |
|||
|
|||
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
|||
private final TbServiceInfoProvider serviceInfoProvider; |
|||
private final TbClusterService clusterService; |
|||
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>(); |
|||
|
|||
@Override |
|||
public ListenableFuture<CommitGitRequest> prepareCommit(TenantId tenantId, VersionCreateRequest request) { |
|||
SettableFuture<CommitGitRequest> future = SettableFuture.create(); |
|||
|
|||
CommitGitRequest commit = new CommitGitRequest(tenantId, request); |
|||
registerAndSend(commit, builder -> builder.setCommitRequest( |
|||
CommitRequestMsg.newBuilder().setPrepareMsg(getCommitPrepareMsg(request)).build() |
|||
).build(), wrap(future, commit)); |
|||
|
|||
return future; |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData) { |
|||
SettableFuture<Void> future = SettableFuture.create(); |
|||
|
|||
String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); |
|||
String entityDataJson; |
|||
try { |
|||
entityDataJson = jsonWriter.writeValueAsString(entityData); |
|||
} catch (IOException e) { |
|||
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
|||
throw new RuntimeException(e); |
|||
} |
|||
|
|||
registerAndSend(commit, builder -> builder.setCommitRequest( |
|||
CommitRequestMsg.newBuilder().setAddMsg( |
|||
TransportProtos.AddMsg.newBuilder() |
|||
.setRelativePath(path).setEntityDataJson(entityDataJson).build() |
|||
).build() |
|||
).build(), wrap(commit.getFuture(), null)); |
|||
return future; |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<Void> deleteAll(CommitGitRequest commit, EntityType entityType) { |
|||
SettableFuture<Void> future = SettableFuture.create(); |
|||
|
|||
String path = getRelativePath(entityType, null); |
|||
|
|||
registerAndSend(commit, builder -> builder.setCommitRequest( |
|||
CommitRequestMsg.newBuilder().setDeleteMsg( |
|||
TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() |
|||
).build() |
|||
).build(), wrap(commit.getFuture(), null)); |
|||
|
|||
return future; |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<VersionCreationResult> push(CommitGitRequest commit) { |
|||
registerAndSend(commit, builder -> builder.setCommitRequest( |
|||
CommitRequestMsg.newBuilder().setPushMsg( |
|||
TransportProtos.PushMsg.newBuilder().build() |
|||
).build() |
|||
).build(), wrap(commit.getFuture())); |
|||
|
|||
return commit.getFuture(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch) { |
|||
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() |
|||
.setBranchName(branch).build()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType) { |
|||
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() |
|||
.setBranchName(branch).setEntityType(entityType.name()) |
|||
.build()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId) { |
|||
return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() |
|||
.setBranchName(branch) |
|||
.setEntityType(entityId.getEntityType().name()) |
|||
.setEntityIdMSB(entityId.getId().getMostSignificantBits()) |
|||
.setEntityIdLSB(entityId.getId().getLeastSignificantBits()) |
|||
.build()); |
|||
} |
|||
|
|||
private ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { |
|||
ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); |
|||
|
|||
registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); |
|||
|
|||
return request.getFuture(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { |
|||
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() |
|||
.setBranchName(branch) |
|||
.setVersionId(versionId) |
|||
.setEntityType(entityType.name()) |
|||
.build()); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { |
|||
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() |
|||
.setBranchName(branch) |
|||
.setVersionId(versionId) |
|||
.build()); |
|||
} |
|||
|
|||
private ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { |
|||
ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); |
|||
|
|||
registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); |
|||
|
|||
return request.getFuture(); |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<String>> listBranches(TenantId tenantId) { |
|||
ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); |
|||
|
|||
registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); |
|||
|
|||
return request.getFuture(); |
|||
} |
|||
|
|||
private <T> void registerAndSend(PendingGitRequest<T> request, Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction, TbQueueCallback callback) { |
|||
if (!request.getFuture().isDone()) { |
|||
pendingRequestMap.putIfAbsent(request.getRequestId(), request); |
|||
clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); |
|||
} else { |
|||
throw new RuntimeException("Future is already done!"); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId) { |
|||
EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); |
|||
registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() |
|||
.setVersionId(versionId) |
|||
.setEntityType(entityId.getEntityType().name()) |
|||
.setEntityIdMSB(entityId.getId().getMostSignificantBits()) |
|||
.setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() |
|||
, wrap(request.getFuture())); |
|||
|
|||
return request.getFuture(); |
|||
// try {
|
|||
// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId,
|
|||
// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId);
|
|||
// return JacksonUtil.fromString(entityDataJson, EntityExportData.class);
|
|||
// } catch (Exception e) {
|
|||
// //TODO: analyze and return meaningful exceptions that we can show to the client;
|
|||
// throw new RuntimeException(e);
|
|||
// }
|
|||
} |
|||
|
|||
@Override |
|||
public ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { |
|||
EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); |
|||
|
|||
registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() |
|||
.setVersionId(versionId) |
|||
.setEntityType(entityType.name()) |
|||
.setOffset(offset) |
|||
.setLimit(limit) |
|||
).build() |
|||
, wrap(request.getFuture())); |
|||
|
|||
return request.getFuture(); |
|||
} |
|||
|
|||
private static <T> TbQueueCallback wrap(SettableFuture<T> future) { |
|||
return new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
future.setException(t); |
|||
} |
|||
}; |
|||
} |
|||
|
|||
private static <T> TbQueueCallback wrap(SettableFuture<T> future, T value) { |
|||
return new TbQueueCallback() { |
|||
@Override |
|||
public void onSuccess(TbQueueMsgMetadata metadata) { |
|||
future.set(value); |
|||
} |
|||
|
|||
@Override |
|||
public void onFailure(Throwable t) { |
|||
future.setException(t); |
|||
} |
|||
}; |
|||
} |
|||
|
|||
private static String getRelativePath(EntityType entityType, EntityId entityId) { |
|||
String path = entityType.name().toLowerCase(); |
|||
if (entityId != null) { |
|||
path += "/" + entityId + ".json"; |
|||
} |
|||
return path; |
|||
} |
|||
|
|||
private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { |
|||
return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); |
|||
} |
|||
|
|||
private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest<?> request) { |
|||
var tenantId = request.getTenantId(); |
|||
var requestId = request.getRequestId(); |
|||
return ToVersionControlServiceMsg.newBuilder() |
|||
.setNodeId(serviceInfoProvider.getServiceId()) |
|||
.setTenantIdMSB(tenantId.getId().getMostSignificantBits()) |
|||
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) |
|||
.setRequestIdMSB(requestId.getMostSignificantBits()) |
|||
.setRequestIdLSB(requestId.getLeastSignificantBits()); |
|||
|
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,37 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
|||
|
|||
import java.util.List; |
|||
|
|||
@Getter |
|||
public class EntitiesContentGitRequest extends PendingGitRequest<List<EntityExportData>> { |
|||
|
|||
private final String versionId; |
|||
private final EntityType entityType; |
|||
|
|||
public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { |
|||
super(tenantId); |
|||
this.versionId = versionId; |
|||
this.entityType = entityType; |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import lombok.Getter; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
|||
|
|||
@Getter |
|||
public class EntityContentGitRequest extends PendingGitRequest<EntityExportData> { |
|||
|
|||
private final String versionId; |
|||
private final EntityId entityId; |
|||
|
|||
public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { |
|||
super(tenantId); |
|||
this.versionId = versionId; |
|||
this.entityId = entityId; |
|||
} |
|||
} |
|||
@ -0,0 +1,57 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import com.google.common.util.concurrent.ListenableFuture; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ExportableEntity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
|||
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
|||
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface GitVersionControlQueueService { |
|||
|
|||
ListenableFuture<CommitGitRequest> prepareCommit(TenantId tenantId, VersionCreateRequest request); |
|||
|
|||
ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData); |
|||
|
|||
ListenableFuture<Void> deleteAll(CommitGitRequest pendingCommit, EntityType entityType); |
|||
|
|||
ListenableFuture<VersionCreationResult> push(CommitGitRequest commit); |
|||
|
|||
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch); |
|||
|
|||
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType); |
|||
|
|||
ListenableFuture<List<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId); |
|||
|
|||
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); |
|||
|
|||
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); |
|||
|
|||
ListenableFuture<List<String>> listBranches(TenantId tenantId); |
|||
|
|||
ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId); |
|||
|
|||
ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); |
|||
|
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
|||
|
|||
import java.util.List; |
|||
|
|||
public class ListBranchesGitRequest extends PendingGitRequest<List<String>> { |
|||
|
|||
public ListBranchesGitRequest(TenantId tenantId) { |
|||
super(tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
|||
|
|||
import java.util.List; |
|||
|
|||
public class ListEntitiesGitRequest extends PendingGitRequest<List<VersionedEntityInfo>> { |
|||
|
|||
public ListEntitiesGitRequest(TenantId tenantId) { |
|||
super(tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
|||
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
|||
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
|||
|
|||
import java.util.List; |
|||
|
|||
public class ListVersionsGitRequest extends PendingGitRequest<List<EntityVersion>> { |
|||
|
|||
public ListVersionsGitRequest(TenantId tenantId) { |
|||
super(tenantId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import com.google.common.util.concurrent.SettableFuture; |
|||
import lombok.Getter; |
|||
import lombok.RequiredArgsConstructor; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
|
|||
import java.util.UUID; |
|||
|
|||
@Getter |
|||
public class PendingGitRequest<T> { |
|||
|
|||
private final UUID requestId; |
|||
private final TenantId tenantId; |
|||
private final SettableFuture<T> future; |
|||
|
|||
public PendingGitRequest(TenantId tenantId) { |
|||
this.requestId = UUID.randomUUID(); |
|||
this.tenantId = tenantId; |
|||
this.future = SettableFuture.create(); |
|||
} |
|||
} |
|||
@ -0,0 +1,116 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.provider; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Component; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; |
|||
import org.thingsboard.server.queue.TbQueueAdmin; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.TbQueueProducer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaAdmin; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaSettings; |
|||
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; |
|||
import org.thingsboard.server.queue.settings.TbQueueCoreSettings; |
|||
import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; |
|||
|
|||
import javax.annotation.PreDestroy; |
|||
|
|||
@Component |
|||
@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") |
|||
public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { |
|||
|
|||
private final TbKafkaSettings kafkaSettings; |
|||
private final TbServiceInfoProvider serviceInfoProvider; |
|||
private final TbQueueCoreSettings coreSettings; |
|||
private final TbQueueVersionControlSettings vcSettings; |
|||
private final TbKafkaConsumerStatsService consumerStatsService; |
|||
|
|||
private final TbQueueAdmin coreAdmin; |
|||
private final TbQueueAdmin vcAdmin; |
|||
private final TbQueueAdmin notificationAdmin; |
|||
|
|||
public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, |
|||
TbServiceInfoProvider serviceInfoProvider, |
|||
TbQueueCoreSettings coreSettings, |
|||
TbQueueVersionControlSettings vcSettings, |
|||
TbKafkaConsumerStatsService consumerStatsService, |
|||
TbKafkaTopicConfigs kafkaTopicConfigs) { |
|||
this.kafkaSettings = kafkaSettings; |
|||
this.serviceInfoProvider = serviceInfoProvider; |
|||
this.coreSettings = coreSettings; |
|||
this.vcSettings = vcSettings; |
|||
this.consumerStatsService = consumerStatsService; |
|||
|
|||
this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); |
|||
this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); |
|||
this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer() { |
|||
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToCoreNotificationMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|||
requestBuilder.settings(kafkaSettings); |
|||
requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); |
|||
requestBuilder.defaultTopic(coreSettings.getTopic()); |
|||
requestBuilder.admin(notificationAdmin); |
|||
return requestBuilder.build(); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueConsumer<TbProtoQueueMsg<ToVersionControlServiceMsg>> createToVersionControlMsgConsumer() { |
|||
TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder<TbProtoQueueMsg<ToVersionControlServiceMsg>> consumerBuilder = TbKafkaConsumerTemplate.builder(); |
|||
consumerBuilder.settings(kafkaSettings); |
|||
consumerBuilder.topic(vcSettings.getTopic()); |
|||
consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); |
|||
consumerBuilder.groupId("tb-vc-node"); |
|||
consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); |
|||
consumerBuilder.admin(vcAdmin); |
|||
consumerBuilder.statsService(consumerStatsService); |
|||
return consumerBuilder.build(); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> createToUsageStatsServiceMsgProducer() { |
|||
TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder<TbProtoQueueMsg<ToUsageStatsServiceMsg>> requestBuilder = TbKafkaProducerTemplate.builder(); |
|||
requestBuilder.settings(kafkaSettings); |
|||
requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); |
|||
requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); |
|||
requestBuilder.admin(coreAdmin); |
|||
return requestBuilder.build(); |
|||
} |
|||
|
|||
@PreDestroy |
|||
private void destroy() { |
|||
if (coreAdmin != null) { |
|||
coreAdmin.destroy(); |
|||
} |
|||
if (vcAdmin != null) { |
|||
vcAdmin.destroy(); |
|||
} |
|||
if (notificationAdmin != null) { |
|||
notificationAdmin.destroy(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,84 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.provider; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.server.gen.transport.TransportProtos; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; |
|||
import org.thingsboard.server.queue.TbQueueProducer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
|
|||
import javax.annotation.PostConstruct; |
|||
|
|||
@Service |
|||
@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") |
|||
public class TbVersionControlProducerProvider implements TbQueueProducerProvider { |
|||
|
|||
private final TbVersionControlQueueFactory tbQueueProvider; |
|||
private TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toTbCoreNotifications; |
|||
private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats; |
|||
|
|||
public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { |
|||
this.tbQueueProvider = tbQueueProvider; |
|||
} |
|||
|
|||
@PostConstruct |
|||
public void init() { |
|||
this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); |
|||
this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> getTransportNotificationsMsgProducer() { |
|||
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineMsg>> getRuleEngineMsgProducer() { |
|||
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToCoreMsg>> getTbCoreMsgProducer() { |
|||
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> getRuleEngineNotificationsMsgProducer() { |
|||
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> getTbCoreNotificationsMsgProducer() { |
|||
return toTbCoreNotifications; |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToVersionControlServiceMsg>> getTbVersionControlMsgProducer() { |
|||
throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); |
|||
} |
|||
|
|||
@Override |
|||
public TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> getTbUsageStatsMsgProducer() { |
|||
return toUsageStats; |
|||
} |
|||
} |
|||
@ -0,0 +1,44 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.provider; |
|||
|
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; |
|||
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; |
|||
import org.thingsboard.server.queue.TbQueueConsumer; |
|||
import org.thingsboard.server.queue.TbQueueProducer; |
|||
import org.thingsboard.server.queue.common.TbProtoQueueMsg; |
|||
|
|||
/** |
|||
* Responsible for initialization of various Producers and Consumers used by TB Version Control Node. |
|||
* Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable |
|||
*/ |
|||
public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { |
|||
|
|||
/** |
|||
* Used to push notifications to other instances of TB Core Service |
|||
* |
|||
* @return |
|||
*/ |
|||
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> createTbCoreNotificationsMsgProducer(); |
|||
|
|||
/** |
|||
* Used to consume messages from TB Core Service |
|||
* |
|||
* @return |
|||
*/ |
|||
TbQueueConsumer<TbProtoQueueMsg<ToVersionControlServiceMsg>> createToVersionControlMsgConsumer(); |
|||
|
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.settings; |
|||
|
|||
import lombok.Data; |
|||
import org.springframework.beans.factory.annotation.Value; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
@Data |
|||
@Component |
|||
public class TbQueueVersionControlSettings { |
|||
|
|||
@Value("${queue.vc.topic:tb_version_control}") |
|||
private String topic; |
|||
|
|||
@Value("${queue.vc.usage-stats-topic:tb_usage_stats}") |
|||
private String usageStatsTopic; |
|||
|
|||
@Value("${queue.vc.partitions:10}") |
|||
private int partitions; |
|||
} |
|||
@ -0,0 +1,26 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.queue.util; |
|||
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; |
|||
|
|||
import java.lang.annotation.Retention; |
|||
import java.lang.annotation.RetentionPolicy; |
|||
|
|||
@Retention(RetentionPolicy.RUNTIME) |
|||
@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") |
|||
public @interface TbVersionControlComponent { |
|||
} |
|||
@ -1,71 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0 |
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
syntax = "proto3"; |
|||
|
|||
option java_package = "org.thingsboard.server.gen.vc.v1"; |
|||
option java_multiple_files = true; |
|||
option java_outer_classname = "EdgeProtos"; |
|||
|
|||
package vc; |
|||
|
|||
// Interface exported by the ThingsBoard Core. |
|||
service TbGitRpcService { |
|||
|
|||
rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} |
|||
|
|||
} |
|||
|
|||
|
|||
/** |
|||
* Data Structures; |
|||
*/ |
|||
message CommitRequestMsg { |
|||
string txId = 1; |
|||
PrepareMsg prepareMsg = 2; |
|||
AddMsg addMsg = 3; |
|||
DeleteMsg deleteMsg = 4; |
|||
PushMsg pushMsg = 5; |
|||
AbortMsg abortMsg = 6; |
|||
} |
|||
|
|||
message CommitResponseMsg { |
|||
string id = 1; |
|||
string name = 2; |
|||
int32 added = 3; |
|||
int32 modified = 4; |
|||
int32 removed = 5; |
|||
} |
|||
|
|||
message PrepareMsg { |
|||
string tenantId = 1; |
|||
string commitMsg = 2; |
|||
string branchName = 3; |
|||
} |
|||
|
|||
message AddMsg { |
|||
string relativePath = 1; |
|||
string entityDataJson = 2; |
|||
} |
|||
|
|||
message DeleteMsg { |
|||
string relativePath = 1; |
|||
} |
|||
|
|||
message PushMsg { |
|||
} |
|||
|
|||
message AbortMsg { |
|||
} |
|||
Loading…
Reference in new issue