diff --git a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java index 8feb898a65..1161403e39 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java @@ -44,10 +44,12 @@ import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.TbResourceInfoFilter; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TbResourceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.util.ThrowingSupplier; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.resource.TbResourceService; @@ -106,13 +108,29 @@ public class TbResourceController extends BaseController { .body(resource); } + @ApiOperation(value = "Download resource (downloadResource)", + notes = "Download resource with a given type and key for the given scope" + AVAILABLE_FOR_ANY_AUTHORIZED_USER) + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + @GetMapping(value = "/resource/{resourceType}/{scope}/{key}") + public ResponseEntity downloadResourceIfChanged(@Parameter(description = "Type of the resource", schema = @Schema(allowableValues = {"lwm2m_model", "jks", "pkcs_12", "js_module", "dashboard"})) + @PathVariable("resourceType") String resourceTypeStr, + @Parameter(description = "Scope of the resource", schema = @Schema(allowableValues = {"system", "tenant"})) + @PathVariable String scope, + @Parameter(description = "Key of the resource, e.g. 'extension.js'") + @PathVariable String key, + @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException { + + ResourceType resourceType = ResourceType.valueOf(resourceTypeStr.toUpperCase()); + return downloadResourceIfChanged(() -> checkResourceInfo(scope, resourceType, key, Operation.READ), etag); + } + @ApiOperation(value = "Download LWM2M Resource (downloadLwm2mResourceIfChanged)", notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @GetMapping(value = "/resource/lwm2m/{resourceId}/download", produces = "application/xml") public ResponseEntity downloadLwm2mResourceIfChanged(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) @PathVariable(RESOURCE_ID) String strResourceId, @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException { - return downloadResourceIfChanged(ResourceType.LWM2M_MODEL, strResourceId, etag); + return downloadResourceIfChanged(strResourceId, etag); } @ApiOperation(value = "Download PKCS_12 Resource (downloadPkcs12ResourceIfChanged)", notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @@ -121,7 +139,7 @@ public class TbResourceController extends BaseController { public ResponseEntity downloadPkcs12ResourceIfChanged(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) @PathVariable(RESOURCE_ID) String strResourceId, @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException { - return downloadResourceIfChanged(ResourceType.PKCS_12, strResourceId, etag); + return downloadResourceIfChanged(strResourceId, etag); } @ApiOperation(value = "Download JKS Resource (downloadJksResourceIfChanged)", @@ -131,7 +149,7 @@ public class TbResourceController extends BaseController { public ResponseEntity downloadJksResourceIfChanged(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) @PathVariable(RESOURCE_ID) String strResourceId, @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException { - return downloadResourceIfChanged(ResourceType.JKS, strResourceId, etag); + return downloadResourceIfChanged(strResourceId, etag); } @ApiOperation(value = "Download JS Resource (downloadJsResourceIfChanged)", notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + AVAILABLE_FOR_ANY_AUTHORIZED_USER) @@ -140,7 +158,7 @@ public class TbResourceController extends BaseController { public ResponseEntity downloadJsResourceIfChanged(@Parameter(description = RESOURCE_ID_PARAM_DESCRIPTION) @PathVariable(RESOURCE_ID) String strResourceId, @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException { - return downloadResourceIfChanged(ResourceType.JS_MODULE, strResourceId, etag); + return downloadResourceIfChanged(strResourceId, etag); } @ApiOperation(value = "Get Resource Info (getResourceInfoById)", @@ -211,6 +229,7 @@ public class TbResourceController extends BaseController { } else { Collections.addAll(resourceTypes, ResourceType.values()); resourceTypes.remove(ResourceType.IMAGE); + resourceTypes.remove(ResourceType.DASHBOARD); } filter.resourceTypes(resourceTypes); if (Authority.SYS_ADMIN.equals(getCurrentUser().getAuthority())) { @@ -271,7 +290,7 @@ public class TbResourceController extends BaseController { @RequestParam String sortOrder, @Parameter(description = SORT_PROPERTY_DESCRIPTION, schema = @Schema(allowableValues = {"id", "name"}, requiredMode = Schema.RequiredMode.REQUIRED)) @RequestParam String sortProperty, - @Parameter(description = "LwM2M Object ids.", array = @ArraySchema(schema = @Schema(type = "string")), required = true) + @Parameter(description = "LwM2M Object ids.", array = @ArraySchema(schema = @Schema(type = "string")), required = true) @RequestParam(required = false) String[] objectIds) throws ThingsboardException { return checkNotNull(tbResourceService.findLwM2mObject(getTenantId(), sortOrder, sortProperty, objectIds)); } @@ -288,30 +307,49 @@ public class TbResourceController extends BaseController { tbResourceService.delete(tbResource, getCurrentUser()); } - private ResponseEntity downloadResourceIfChanged(ResourceType resourceType, String strResourceId, String etag) throws ThingsboardException { + private ResponseEntity downloadResourceIfChanged(String strResourceId, String etag) throws ThingsboardException { checkParameter(RESOURCE_ID, strResourceId); TbResourceId resourceId = new TbResourceId(toUUID(strResourceId)); + return downloadResourceIfChanged(() -> checkResourceInfoId(resourceId, Operation.READ), etag); + } + + private ResponseEntity downloadResourceIfChanged(ThrowingSupplier resourceInfoProvider, + String etag) throws ThingsboardException { + TbResourceInfo resourceInfo = resourceInfoProvider.get(); if (etag != null) { - TbResourceInfo tbResourceInfo = checkResourceInfoId(resourceId, Operation.READ); etag = StringUtils.remove(etag, '\"'); // etag is wrapped in double quotes due to HTTP specification - if (etag.equals(tbResourceInfo.getEtag())) { + if (etag.equals(resourceInfo.getEtag())) { return ResponseEntity.status(HttpStatus.NOT_MODIFIED) - .eTag(tbResourceInfo.getEtag()) + .eTag(resourceInfo.getEtag()) .build(); } } - TbResource tbResource = checkResourceId(resourceId, Operation.READ); - - ByteArrayResource resource = new ByteArrayResource(tbResource.getData()); + byte[] data = resourceService.getResourceData(resourceInfo.getTenantId(), resourceInfo.getId()); + ByteArrayResource resource = new ByteArrayResource(data); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + tbResource.getFileName()) - .header("x-filename", tbResource.getFileName()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + resourceInfo.getFileName()) + .header("x-filename", resourceInfo.getFileName()) .contentLength(resource.contentLength()) - .header("Content-Type", resourceType.getMediaType()) + .header("Content-Type", resourceInfo.getResourceType().getMediaType()) .cacheControl(CacheControl.noCache()) - .eTag(tbResource.getEtag()) + .eTag(resourceInfo.getEtag()) .body(resource); } + private TbResourceInfo checkResourceInfo(String scope, ResourceType resourceType, String key, Operation operation) throws ThingsboardException { + TenantId tenantId; + if (scope.equals("tenant")) { + tenantId = getTenantId(); + } else if (scope.equals("system")) { + tenantId = TenantId.SYS_TENANT_ID; + } else { + throw new IllegalArgumentException("Invalid scope"); + } + + TbResourceInfo resourceInfo = resourceService.findResourceInfoByTenantIdAndKey(tenantId, resourceType, key); + checkEntity(getCurrentUser(), checkNotNull(resourceInfo), operation); + return resourceInfo; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java new file mode 100644 index 0000000000..934cb13246 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java @@ -0,0 +1,126 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.dashboard; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.resource.ResourceService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService; +import org.thingsboard.server.service.sync.GitSyncService; +import org.thingsboard.server.service.sync.vc.GitRepository.FileType; +import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DashboardSyncService { + + private final GitSyncService gitSyncService; + private final ResourceService resourceService; + private final TbWidgetsBundleService tbWidgetsBundleService; + private final PartitionService partitionService; + + @Value("${transport.gateway.dashboard.sync.enabled:true}") + private boolean enabled; + @Value("${transport.gateway.dashboard.sync.repository_url:}") + private String repoUrl; + @Value("${transport.gateway.dashboard.sync.fetch_frequency:24}") + private int fetchFrequencyHours; + + private static final String REPO_KEY = "gateways-dashboard"; + private static final String GATEWAY_RESOURCE_ID_PARAM = "${GATEWAY_RESOURCE_ID}"; + private static final String GATEWAYS_DASHBOARD_KEY = "gateways_dashboard.json"; + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void init() throws Exception { + if (!enabled) { + return; + } + gitSyncService.registerSync(REPO_KEY, repoUrl, "main", TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update); + } + + private void update() { + if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) { + return; + } + + RepoFile extensionResourceFile = listFiles("resources").get(0); + String data = getFileContent(extensionResourceFile.path()); + TbResource extensionResource = createOrUpdateResource(ResourceType.JS_MODULE, extensionResourceFile.name(), data.getBytes(StandardCharsets.UTF_8)); + String extensionResourceId = extensionResource.getUuidId().toString(); + + Stream widgetsBundles = listFiles("widget_bundles").stream() + .map(widgetsBundleFile -> { + String widgetsBundleDescriptor = getFileContent(widgetsBundleFile.path()); + widgetsBundleDescriptor = widgetsBundleDescriptor.replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId); + return JacksonUtil.toJsonNode(widgetsBundleDescriptor); + }); + Stream widgetTypes = listFiles("widget_types").stream() + .map(widgetTypeFile -> { + String widgetTypeDetails = getFileContent(widgetTypeFile.path()); + widgetTypeDetails = widgetTypeDetails.replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId); + return JacksonUtil.toJsonNode(widgetTypeDetails); + }); + tbWidgetsBundleService.updateWidgets(TenantId.SYS_TENANT_ID, widgetsBundles, widgetTypes); + + RepoFile dashboardFile = listFiles("dashboards").get(0); + String dashboardJson = getFileContent(dashboardFile.path()).replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId); + createOrUpdateResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, dashboardJson.getBytes(StandardCharsets.UTF_8)); + + log.info("Gateways dashboard sync completed"); + } + + private TbResource createOrUpdateResource(ResourceType resourceType, String resourceKey, byte[] data) { + TbResource resource = resourceService.findResourceByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey); + if (resource == null) { + resource = new TbResource(); + resource.setTenantId(TenantId.SYS_TENANT_ID); + resource.setResourceType(resourceType); + resource.setResourceKey(resourceKey); + resource.setFileName(resourceKey); + resource.setTitle(resourceKey); + } + resource.setData(data); + log.debug("{} resource {}", (resource.getId() == null ? "Creating" : "Updating"), resourceKey); + return resourceService.saveResource(resource); + } + + private List listFiles(String path) { + return gitSyncService.listFiles(REPO_KEY, path, 1, FileType.FILE); + } + + private String getFileContent(String path) { + return gitSyncService.getFileContent(REPO_KEY, path); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java index 92896c068e..352c8caee2 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java @@ -15,25 +15,34 @@ */ package org.thingsboard.server.service.entitiy.widgets.bundle; +import com.fasterxml.jackson.databind.JsonNode; import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.widget.WidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; @Service @TbCoreComponent @AllArgsConstructor +@Slf4j public class DefaultWidgetsBundleService extends AbstractTbEntityService implements TbWidgetsBundleService { private final WidgetsBundleService widgetsBundleService; @@ -79,4 +88,69 @@ public class DefaultWidgetsBundleService extends AbstractTbEntityService impleme widgetTypeService.updateWidgetsBundleWidgetFqns(user.getTenantId(), widgetsBundleId, widgetFqns); autoCommit(user, widgetsBundleId); } + + @Transactional + @Override + public void updateWidgets(TenantId tenantId, Stream bundles, Stream widgets) { + widgets.forEach(widgetTypeJson -> { + try { + WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class); + WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(tenantId, widgetTypeDetails.getFqn()); + if (existingWidget != null) { + widgetTypeDetails.setId(existingWidget.getId()); + widgetTypeDetails.setCreatedTime(existingWidget.getCreatedTime()); + } + widgetTypeDetails.setTenantId(tenantId); + widgetTypeService.saveWidgetType(widgetTypeDetails); + log.debug("{} widget type {}", existingWidget == null ? "Created" : "Updated", widgetTypeDetails.getFqn()); + } catch (Exception e) { + throw new RuntimeException("Unable to load widget type from json: " + widgetTypeJson, e); + } + }); + + bundles.forEach(widgetsBundleDescriptorJson -> { + if (widgetsBundleDescriptorJson == null || !widgetsBundleDescriptorJson.has("widgetsBundle")) { + throw new RuntimeException("Invalid widgets bundle json: [" + widgetsBundleDescriptorJson + "]"); + } + + JsonNode widgetsBundleJson = widgetsBundleDescriptorJson.get("widgetsBundle"); + WidgetsBundle widgetsBundle = JacksonUtil.treeToValue(widgetsBundleJson, WidgetsBundle.class); + WidgetsBundle existingWidgetsBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(tenantId, widgetsBundle.getAlias()); + if (existingWidgetsBundle != null) { + widgetsBundle.setId(existingWidgetsBundle.getId()); + widgetsBundle.setCreatedTime(existingWidgetsBundle.getCreatedTime()); + } + widgetsBundle.setTenantId(tenantId); + widgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); + log.debug("{} widgets bundle {}", existingWidgetsBundle == null ? "Created" : "Updated", widgetsBundle.getAlias()); + + List widgetTypeFqns = new ArrayList<>(); + if (widgetsBundleDescriptorJson.has("widgetTypes")) { + JsonNode widgetTypesArrayJson = widgetsBundleDescriptorJson.get("widgetTypes"); + widgetTypesArrayJson.forEach(widgetTypeJson -> { + try { + WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class); + WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(tenantId, widgetTypeDetails.getFqn()); + if (existingWidget != null) { + widgetTypeDetails.setId(existingWidget.getId()); + widgetTypeDetails.setCreatedTime(existingWidget.getCreatedTime()); + } + widgetTypeDetails.setTenantId(tenantId); + widgetTypeDetails = widgetTypeService.saveWidgetType(widgetTypeDetails); + widgetTypeFqns.add(widgetTypeDetails.getFqn()); + } catch (Exception e) { + throw new RuntimeException("Unable to load widget type from json: " + widgetsBundleDescriptorJson, e); + } + }); + } + if (widgetsBundleDescriptorJson.has("widgetTypeFqns")) { + JsonNode widgetFqnsArrayJson = widgetsBundleDescriptorJson.get("widgetTypeFqns"); + widgetFqnsArrayJson.forEach(fqnJson -> { + widgetTypeFqns.add(fqnJson.asText()); + }); + } + widgetTypeService.updateWidgetsBundleWidgetFqns(tenantId, widgetsBundle.getId(), widgetTypeFqns); + }); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java index db03701fcf..6a270b5ebe 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java @@ -15,13 +15,16 @@ */ package org.thingsboard.server.service.entitiy.widgets.bundle; +import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.service.entitiy.SimpleTbEntityService; import java.util.List; +import java.util.stream.Stream; public interface TbWidgetsBundleService extends SimpleTbEntityService { @@ -29,4 +32,6 @@ public interface TbWidgetsBundleService extends SimpleTbEntityService widgetFqns, User user) throws Exception; + void updateWidgets(TenantId tenantId, Stream bundles, Stream widgets); + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 7f76ce860f..a57c28270b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.install; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Streams; import lombok.Getter; import lombok.Setter; import lombok.SneakyThrows; @@ -37,14 +38,9 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.OAuth2ClientRegistrationTemplate; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.widget.DeprecatedFilter; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import org.thingsboard.server.common.data.widget.WidgetTypeInfo; -import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService; @@ -53,7 +49,7 @@ import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.util.ImageUtils; import org.thingsboard.server.dao.widget.WidgetTypeService; -import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService; import org.thingsboard.server.service.install.update.ImagesUpdater; import java.io.IOException; @@ -62,9 +58,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.function.Function; import java.util.stream.Stream; @@ -114,7 +108,7 @@ public class InstallScripts { private WidgetTypeService widgetTypeService; @Autowired - private WidgetsBundleService widgetsBundleService; + private TbWidgetsBundleService tbWidgetsBundleService; @Autowired private OAuth2ConfigTemplateService oAuth2TemplateService; @@ -216,91 +210,39 @@ public class InstallScripts { public void loadSystemWidgets() throws Exception { log.info("Loading system widgets"); - Map widgetsBundlesMap = new HashMap<>(); + Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR); + Stream bundles; try (DirectoryStream dirStream = Files.newDirectoryStream(widgetBundlesDir, path -> path.toString().endsWith(JSON_EXT))) { - dirStream.forEach( - path -> { - JsonNode widgetsBundleDescriptorJson; - try { - widgetsBundleDescriptorJson = JacksonUtil.toJsonNode(path.toFile()); - } catch (Exception e) { - log.error("Unable to parse widgets bundle from json: [{}]", path); - throw new RuntimeException("Unable to parse widgets bundle from json", e); - } - if (widgetsBundleDescriptorJson == null || !widgetsBundleDescriptorJson.has("widgetsBundle")) { - log.error("Invalid widgets bundle json: [{}]", path); - throw new RuntimeException("Invalid widgets bundle json: [" + path + "]"); - } - widgetsBundlesMap.put(path, widgetsBundleDescriptorJson); - JsonNode bundleAliasNode = widgetsBundleDescriptorJson.get("widgetsBundle").get("alias"); - if (bundleAliasNode == null || !bundleAliasNode.isTextual()) { - log.error("Invalid widgets bundle json: [{}]", path); - throw new RuntimeException("Invalid widgets bundle json: [" + path + "]"); - } - String bundleAlias = bundleAliasNode.asText(); - try { - this.deleteSystemWidgetBundle(bundleAlias); - } catch (Exception e) { - log.error("Failed to delete system widgets bundle: [{}]", bundleAlias); - throw new RuntimeException("Failed to delete system widgets bundle: [" + bundleAlias + "]", e); - } - } - ); + bundles = Streams.stream(dirStream).map(path -> { + try { + return JacksonUtil.toJsonNode(path.toFile()); + } catch (Exception e) { + log.error("Unable to parse widgets bundle from json: [{}]", path); + throw new RuntimeException("Unable to parse widgets bundle from json", e); + } + }); + } + Stream widgets; Path widgetTypesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR); if (Files.exists(widgetTypesDir)) { try (DirectoryStream dirStream = Files.newDirectoryStream(widgetTypesDir, path -> path.toString().endsWith(JSON_EXT))) { - dirStream.forEach( - path -> { - try { - JsonNode widgetTypeJson = JacksonUtil.toJsonNode(path.toFile()); - WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class); - widgetTypeService.saveWidgetType(widgetTypeDetails); - } catch (Exception e) { - log.error("Unable to load widget type from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load widget type from json", e); - } - } - ); - } - } - this.loadSystemScadaSymbols(); - for (var widgetsBundleDescriptorEntry : widgetsBundlesMap.entrySet()) { - Path path = widgetsBundleDescriptorEntry.getKey(); - try { - JsonNode widgetsBundleDescriptorJson = widgetsBundleDescriptorEntry.getValue(); - JsonNode widgetsBundleJson = widgetsBundleDescriptorJson.get("widgetsBundle"); - WidgetsBundle widgetsBundle = JacksonUtil.treeToValue(widgetsBundleJson, WidgetsBundle.class); - WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); - List widgetTypeFqns = new ArrayList<>(); - if (widgetsBundleDescriptorJson.has("widgetTypes")) { - JsonNode widgetTypesArrayJson = widgetsBundleDescriptorJson.get("widgetTypes"); - widgetTypesArrayJson.forEach( - widgetTypeJson -> { - try { - WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class); - var savedWidgetType = widgetTypeService.saveWidgetType(widgetTypeDetails); - widgetTypeFqns.add(savedWidgetType.getFqn()); - } catch (Exception e) { - log.error("Unable to load widget type from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load widget type from json", e); - } - } - ); - } - if (widgetsBundleDescriptorJson.has("widgetTypeFqns")) { - JsonNode widgetFqnsArrayJson = widgetsBundleDescriptorJson.get("widgetTypeFqns"); - widgetFqnsArrayJson.forEach(fqnJson -> { - widgetTypeFqns.add(fqnJson.asText()); - }); - } - widgetTypeService.updateWidgetsBundleWidgetFqns(TenantId.SYS_TENANT_ID, savedWidgetsBundle.getId(), widgetTypeFqns); - } catch (Exception e) { - log.error("Unable to load widgets bundle from json: [{}]", path.toString()); - throw new RuntimeException("Unable to load widgets bundle from json", e); + widgets = Streams.stream(dirStream).map(path -> { + try { + return JacksonUtil.toJsonNode(path.toFile()); + } catch (Exception e) { + log.error("Unable to parse widget type from json: [{}]", path); + throw new RuntimeException("Unable to parse widget type from json", e); + } + }); } + } else { + widgets = Stream.empty(); } + tbWidgetsBundleService.updateWidgets(TenantId.SYS_TENANT_ID, bundles, widgets); + + loadSystemScadaSymbols(); } private void loadSystemScadaSymbols() throws Exception { @@ -355,7 +297,7 @@ public class InstallScripts { } private WidgetTypeDetails saveScadaSymbolWidget(WidgetTypeDetails template, TbResourceInfo scadaSymbol, - ImageUtils.ScadaSymbolMetadataInfo metadata) { + ImageUtils.ScadaSymbolMetadataInfo metadata) { String symbolUrl = DataConstants.TB_IMAGE_PREFIX + scadaSymbol.getLink(); WidgetTypeDetails scadaSymbolWidget = new WidgetTypeDetails(); JsonNode descriptor = JacksonUtil.clone(template.getDescriptor()); @@ -375,38 +317,22 @@ public class InstallScripts { defaultConfig.put("title", metadata.getTitle()); ObjectNode settings; if (defaultConfig.has("settings")) { - settings = (ObjectNode)defaultConfig.get("settings"); + settings = (ObjectNode) defaultConfig.get("settings"); } else { settings = JacksonUtil.newObjectNode(); defaultConfig.set("settings", settings); } settings.put("scadaSymbolUrl", symbolUrl); - ((ObjectNode)descriptor).put("defaultConfig", JacksonUtil.toString(defaultConfig)); - ((ObjectNode)descriptor).put("sizeX", metadata.getWidgetSizeX()); - ((ObjectNode)descriptor).put("sizeY", metadata.getWidgetSizeY()); + ((ObjectNode) descriptor).put("defaultConfig", JacksonUtil.toString(defaultConfig)); + ((ObjectNode) descriptor).put("sizeX", metadata.getWidgetSizeX()); + ((ObjectNode) descriptor).put("sizeY", metadata.getWidgetSizeY()); String controllerScript = descriptor.get("controllerScript").asText(); controllerScript = controllerScript.replaceAll("previewWidth: '\\d*px'", "previewWidth: '" + (metadata.getWidgetSizeX() * 100) + "px'"); controllerScript = controllerScript.replaceAll("previewHeight: '\\d*px'", "previewHeight: '" + (metadata.getWidgetSizeY() * 100 + 20) + "px'"); - ((ObjectNode)descriptor).put("controllerScript", controllerScript); + ((ObjectNode) descriptor).put("controllerScript", controllerScript); return widgetTypeService.saveWidgetType(scadaSymbolWidget); } - private void deleteSystemWidgetBundle(String bundleAlias) { - WidgetsBundle widgetsBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, bundleAlias); - if (widgetsBundle != null) { - PageData widgetTypes; - var pageLink = new PageLink(1024); - do { - widgetTypes = widgetTypeService.findWidgetTypesInfosByWidgetsBundleId(TenantId.SYS_TENANT_ID, widgetsBundle.getId(), false, DeprecatedFilter.ALL, null, pageLink); - for (var widgetType : widgetTypes.getData()) { - widgetTypeService.deleteWidgetType(TenantId.SYS_TENANT_ID, widgetType.getId()); - } - pageLink.nextPageLink(); - } while (widgetTypes.hasNext()); - widgetsBundleService.deleteWidgetsBundle(TenantId.SYS_TENANT_ID, widgetsBundle.getId()); - } - } - public void updateImages() { imagesUpdater.updateWidgetsBundlesImages(); imagesUpdater.updateWidgetTypesImages(); @@ -419,7 +345,7 @@ public class InstallScripts { public void loadSystemImages() { log.info("Loading system images..."); Stream dashboardsFiles = Stream.concat(Files.list(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR)), - Files.list(Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DASHBOARDS_DIR))); + Files.list(Paths.get(getDataDir(), JSON_DIR, TENANT_DIR, DASHBOARDS_DIR))); try (dashboardsFiles) { dashboardsFiles.forEach(file -> { try { @@ -534,4 +460,5 @@ public class InstallScripts { resourceService.saveResource(resource); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java new file mode 100644 index 0000000000..7714cf80fe --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java @@ -0,0 +1,168 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync; + +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.GitRepository; +import org.thingsboard.server.service.sync.vc.GitRepository.FileType; +import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; + +import java.net.URI; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultGitSyncService implements GitSyncService { + + @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync")); + private final Map repositories = new ConcurrentHashMap<>(); + private final Map updateListeners = new ConcurrentHashMap<>(); + + @Override + public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) { + RepositorySettings settings = new RepositorySettings(); + settings.setRepositoryUri(repoUri); + settings.setDefaultBranch(branch); + if (onUpdate != null) { + updateListeners.put(key, onUpdate); + } + + executor.execute(() -> { + initRepository(key, settings); + }); + + executor.scheduleWithFixedDelay(() -> { + GitRepository repository = repositories.get(key); + if (repository == null || !GitRepository.exists(repository.getDirectory())) { + initRepository(key, settings); + return; + } + + try { + log.debug("[{}] Fetching repository", key); + repository.fetch(); + onUpdate(key); + } catch (Throwable e) { + log.error("[{}] Failed to fetch repository", key, e); + } + }, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS); + } + + @Override + public List listFiles(String key, String path, int depth, FileType type) { + GitRepository repository = getRepository(key); + return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream() + .filter(file -> type == null || file.type() == type) + .toList(); + } + + + @Override + public String getFileContent(String key, String path) { + GitRepository repository = getRepository(key); + try { + return repository.getFileContentAtCommit(path, getBranchRef(repository)); + } catch (Exception e) { + log.warn("[{}] Failed to get file content for path {}: {}", key, path, e.getMessage()); + return "{}"; + } + } + + @Override + public String getGithubRawContentUrl(String key, String path) { + if (path == null) { + return ""; + } + RepositorySettings settings = getRepository(key).getSettings(); + return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true"; + } + + private GitRepository getRepository(String key) { + GitRepository repository = repositories.get(key); + if (repository != null) { + if (!GitRepository.exists(repository.getDirectory())) { + // reinitializing the repository because folder was deleted + initRepository(key, repository.getSettings()); + } + } + + repository = repositories.get(key); + if (repository == null) { + throw new IllegalStateException(key + " repository is not initialized"); + } + return repository; + } + + private void initRepository(String key, RepositorySettings settings) { + try { + repositories.remove(key); + Path directory = getRepoDirectory(settings); + + GitRepository repository = GitRepository.openOrClone(directory, settings, true); + repositories.put(key, repository); + log.info("[{}] Initialized repository", key); + + onUpdate(key); + } catch (Throwable e) { + log.error("[{}] Failed to initialize repository with settings {}", key, settings, e); + } + } + + private void onUpdate(String key) { + Runnable listener = updateListeners.get(key); + if (listener != null) { + log.debug("[{}] Handling repository update", key); + try { + listener.run(); + } catch (Throwable e) { + log.error("[{}] Failed to handle repository update", key, e); + } + } + } + + private Path getRepoDirectory(RepositorySettings settings) { + // using uri to define folder name in case repo url is changed + String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", ""); + return Path.of(repositoriesFolder, name); + } + + private String getBranchRef(GitRepository repository) { + return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch(); + } + + @PreDestroy + private void preDestroy() { + executor.shutdownNow(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java b/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java new file mode 100644 index 0000000000..d1a09757f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync; + +import org.thingsboard.server.service.sync.vc.GitRepository.FileType; +import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile; + +import java.util.List; + +public interface GitSyncService { + + void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate); + + List listFiles(String key, String path, int depth, FileType type); + + String getFileContent(String key, String path); + + String getGithubRawContentUrl(String key, String path); + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e9c8668958..ccf556d3a8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1244,6 +1244,15 @@ transport: enabled: "${TB_TRANSPORT_STATS_ENABLED:true}" # Interval of transport statistics logging print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" + gateway: + dashboard: + sync: + # Enable/disable gateways dashboard sync with git repository + enabled: "${TB_GATEWAY_DASHBOARD_SYNC_ENABLED:true}" + # URL of gateways dashboard repository + repository_url: "${TB_GATEWAY_DASHBOARD_SYNC_REPOSITORY_URL:https://github.com/thingsboard/gateway-management-extensions-dist.git}" + # Fetch frequency in hours for gateways dashboard repository + fetch_frequency: "${TB_GATEWAY_DASHBOARD_SYNC_FETCH_FREQUENCY:24}" # CoAP server parameters coap: diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncServiceTest.java new file mode 100644 index 0000000000..07e6ff474c --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncServiceTest.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.entitiy.dashboard; + +import org.junit.Test; +import org.springframework.mock.web.MockHttpServletResponse; +import org.springframework.test.context.TestPropertySource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +@TestPropertySource(properties = { + "transport.gateway.dashboard.sync.enabled=true" +}) +public class DashboardSyncServiceTest extends AbstractControllerTest { + + @Test + public void testGatewaysDashboardSync() throws Exception { + loginTenantAdmin(); + await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { + MockHttpServletResponse response = doGet("/api/resource/dashboard/system/gateways_dashboard.json") + .andExpect(status().isOk()) + .andReturn().getResponse(); + String dashboardJson = response.getContentAsString(); + String etag = response.getHeader("ETag"); + + Dashboard dashboard = JacksonUtil.fromString(dashboardJson, Dashboard.class); + assertThat(dashboard).isNotNull(); + assertThat(dashboard.getTitle()).containsIgnoringCase("gateway"); + assertThat(etag).isNotBlank(); + }); + } + +} \ No newline at end of file diff --git a/application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java b/application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java index 37b50fda4f..e63e80ef3d 100644 --- a/application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java +++ b/application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java @@ -36,7 +36,7 @@ import org.thingsboard.server.dao.service.validator.RuleChainDataValidator; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.usagerecord.ApiLimitService; import org.thingsboard.server.dao.widget.WidgetTypeService; -import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService; import org.thingsboard.server.service.install.update.ImagesUpdater; import java.io.IOException; @@ -60,7 +60,7 @@ class InstallScriptsTest { @MockBean WidgetTypeService widgetTypeService; @MockBean - WidgetsBundleService widgetsBundleService; + TbWidgetsBundleService tbWidgetsBundleService; @MockBean OAuth2ConfigTemplateService oAuth2TemplateService; @MockBean diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 92959e4552..9951caa876 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -54,3 +54,5 @@ sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 server.log_controller_error_stack_trace=false + +transport.gateway.dashboard.sync.enabled=false diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index f43144a24e..a9ac276f4d 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -17,7 +17,9 @@ - + + + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java index b1df83e3d8..7b9c20211f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java @@ -38,6 +38,8 @@ public interface ResourceService extends EntityDaoService { TbResource findResourceById(TenantId tenantId, TbResourceId resourceId); + byte[] getResourceData(TenantId tenantId, TbResourceId resourceId); + TbResourceInfo findResourceInfoById(TenantId tenantId, TbResourceId resourceId); TbResourceInfo findResourceInfoByTenantIdAndKey(TenantId tenantId, ResourceType resourceType, String resourceKey); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java index 39a5210849..34b27a7504 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeService.java @@ -65,4 +65,6 @@ public interface WidgetTypeService extends EntityDaoService { void deleteWidgetTypesByTenantId(TenantId tenantId); + void deleteWidgetTypesByBundleId(TenantId tenantId, WidgetsBundleId bundleId); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java index 32e354ec2c..f448440c5a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ResourceType.java @@ -24,7 +24,8 @@ public enum ResourceType { JKS("application/x-java-keystore", false, false), PKCS_12("application/x-pkcs12", false, false), JS_MODULE("application/javascript", true, true), - IMAGE(null, true, true); + IMAGE(null, true, true), + DASHBOARD("application/json", true, true); @Getter private final String mediaType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java index 6353d73c17..0561869730 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.common.data.util; +import org.thingsboard.server.common.data.exception.ThingsboardException; + @FunctionalInterface public interface ThrowingSupplier { - T get() throws Exception; + T get() throws ThingsboardException; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 2aa31e194d..239eb18748 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -40,7 +40,6 @@ import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; import java.util.List; @@ -283,23 +282,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { private GitRepository openOrCloneRepository(TenantId tenantId, RepositorySettings settings, boolean fetch) throws Exception { log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, settings.isLocalOnly() ? "local_" + settings.getRepositoryUri() : tenantId.getId().toString()); - - GitRepository repository; - if (GitRepository.exists(repositoryDirectory.toString())) { - repository = GitRepository.open(repositoryDirectory.toFile(), settings); - if (fetch) { - repository.fetch(); - } - } else { - FileUtils.deleteDirectory(repositoryDirectory.toFile()); - Files.createDirectories(repositoryDirectory); - if (settings.isLocalOnly()) { - repository = GitRepository.create(settings, repositoryDirectory.toFile()); - } else { - repository = GitRepository.clone(settings, repositoryDirectory.toFile()); - } - } - + GitRepository repository = GitRepository.openOrClone(repositoryDirectory, settings, fetch); repositories.put(tenantId, repository); log.debug("[{}] Init tenant repository completed.", tenantId); return repository; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 2ff6a97c21..1f434c1447 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -138,6 +138,25 @@ public class GitRepository { return new GitRepository(git, settings, authHandler, directory.getAbsolutePath()); } + public static GitRepository openOrClone(Path directory, RepositorySettings settings, boolean fetch) throws IOException, GitAPIException { + GitRepository repository; + if (GitRepository.exists(directory.toString())) { + repository = GitRepository.open(directory.toFile(), settings); + if (fetch) { + repository.fetch(); + } + } else { + FileUtils.deleteDirectory(directory.toFile()); + Files.createDirectories(directory); + if (settings.isLocalOnly()) { + repository = GitRepository.create(settings, directory.toFile()); + } else { + repository = GitRepository.clone(settings, directory.toFile()); + } + } + return repository; + } + public static void test(RepositorySettings settings, File directory) throws Exception { if (settings.isLocalOnly()) { return; @@ -235,22 +254,29 @@ public class GitRepository { return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); } - public List listFilesAtCommit(String commitId) throws IOException { - return listFilesAtCommit(commitId, null); + public List listFilesAtCommit(String commitId, String path) { + return listFilesAtCommit(commitId, path, -1).stream().map(RepoFile::path).toList(); } - public List listFilesAtCommit(String commitId, String path) throws IOException { + @SneakyThrows + public List listFilesAtCommit(String commitId, String path, int depth) { log.debug("Executing listFilesAtCommit [{}][{}][{}]", settings.getRepositoryUri(), commitId, path); - List files = new ArrayList<>(); + List files = new ArrayList<>(); RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) { treeWalk.setFilter(PathFilter.create(path)); } - treeWalk.setRecursive(true); + boolean fixedDepth = depth != -1; + treeWalk.setRecursive(!fixedDepth); while (treeWalk.next()) { - files.add(treeWalk.getPathString()); + if (!fixedDepth || treeWalk.getDepth() == depth) { + files.add(new RepoFile(treeWalk.getPathString(), treeWalk.getNameString(), treeWalk.isSubtree() ? FileType.DIRECTORY : FileType.FILE)); + } + if (fixedDepth && treeWalk.getDepth() < depth) { + treeWalk.enterSubtree(); + } } } return files; @@ -592,4 +618,10 @@ public class GitRepository { private String diffStringValue; } + public record RepoFile(String path, String name, FileType type) {} + + public enum FileType { + FILE, DIRECTORY + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index 327f32f5a5..dc4b6c28fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -248,8 +248,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic @Override public byte[] getImageData(TenantId tenantId, TbResourceId imageId) { - log.trace("Executing getImageData [{}] [{}]", tenantId, imageId); - return resourceDao.getResourceData(tenantId, imageId); + return getResourceData(tenantId, imageId); } @Override @@ -448,7 +447,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic mdResourceName = new String(Base64.getDecoder().decode(matcher.group(2)), StandardCharsets.UTF_8); if (StringUtils.isNotBlank(matcher.group(3))) { mdResourceSubType = new String(Base64.getDecoder().decode(matcher.group(3)), StandardCharsets.UTF_8); - }; + } mdMediaType = matcher.group(4); } else if (data.startsWith(DataConstants.TB_IMAGE_PREFIX + "data:image/") || (!strict && data.startsWith("data:image/"))) { mdMediaType = StringUtils.substringBetween(data, "data:", ";base64"); @@ -687,4 +686,5 @@ public class BaseImageService extends BaseResourceService implements ImageServic private final boolean updated; private final String value; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index fcace55c93..a5865e6fbb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -120,6 +120,12 @@ public class BaseResourceService extends AbstractCachedEntityService toRemove = existingBundleWidgets.stream() .map(WidgetsBundleWidget::getWidgetTypeId) .filter(widgetTypeId -> bundleWidgets.stream().noneMatch(newBundleWidget -> - newBundleWidget.getWidgetTypeId().equals(widgetTypeId))).collect(Collectors.toList()); + newBundleWidget.getWidgetTypeId().equals(widgetTypeId))).toList(); for (WidgetTypeId widgetTypeId : toRemove) { widgetTypeDao.removeWidgetTypeFromWidgetsBundle(widgetsBundleId.getId(), widgetTypeId.getId()); } @@ -243,6 +242,12 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { tenantWidgetTypeRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteWidgetTypesByBundleId(TenantId tenantId, WidgetsBundleId bundleId) { + log.trace("Executing deleteWidgetTypesByBundleId, tenantId [{}], bundleId [{}]", tenantId, bundleId); + bundleWidgetTypesRemover.removeEntities(tenantId, bundleId); + } + @Override public void deleteByTenantId(TenantId tenantId) { deleteWidgetTypesByTenantId(tenantId); @@ -258,24 +263,36 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { return EntityType.WIDGET_TYPE; } - private PaginatedRemover tenantWidgetTypeRemover = - new PaginatedRemover<>() { - - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return widgetTypeDao.findTenantWidgetTypesByTenantId( - WidgetTypeFilter.builder() - .tenantId(id) - .fullSearch(false) - .deprecatedFilter(DeprecatedFilter.ALL) - .widgetTypes(null).build(), - pageLink); - } - - @Override - protected void removeEntity(TenantId tenantId, WidgetTypeInfo entity) { - deleteWidgetType(tenantId, new WidgetTypeId(entity.getUuidId())); - } - }; + private final PaginatedRemover tenantWidgetTypeRemover = new PaginatedRemover<>() { + + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return widgetTypeDao.findTenantWidgetTypesByTenantId( + WidgetTypeFilter.builder() + .tenantId(id) + .fullSearch(false) + .deprecatedFilter(DeprecatedFilter.ALL) + .widgetTypes(null).build(), + pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, WidgetTypeInfo entity) { + deleteWidgetType(tenantId, new WidgetTypeId(entity.getUuidId())); + } + }; + + private final PaginatedRemover bundleWidgetTypesRemover = new PaginatedRemover<>() { + + @Override + protected PageData findEntities(TenantId tenantId, WidgetsBundleId widgetsBundleId, PageLink pageLink) { + return findWidgetTypesInfosByWidgetsBundleId(tenantId, widgetsBundleId, false, DeprecatedFilter.ALL, null, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, WidgetTypeInfo widgetTypeInfo) { + deleteWidgetType(tenantId, widgetTypeInfo.getId()); + } + }; }