Browse Source

Merge branch 'feature/dashboard-sync' into feature/4661-gateway-sync

pull/11872/head^2
Max Petrov 2 years ago
committed by GitHub
parent
commit
04346bee67
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 10
      application/src/main/data/json/system/widget_types/gateway_configuration.json
  2. 10
      application/src/main/data/json/system/widget_types/gateway_configuration__single_device_.json
  3. 10
      application/src/main/data/json/system/widget_types/gateway_connectors.json
  4. 10
      application/src/main/data/json/system/widget_types/gateway_custom_statistics.json
  5. 10
      application/src/main/data/json/system/widget_types/gateway_general_chart_statistics.json
  6. 10
      application/src/main/data/json/system/widget_types/gateway_general_configuration.json
  7. 10
      application/src/main/data/json/system/widget_types/gateway_logs.json
  8. 10
      application/src/main/data/json/system/widget_types/service_rpc.json
  9. 7031
      application/src/main/data/resources/dashboards/gateways_dashboard.json
  10. 1
      application/src/main/data/resources/js_modules/gateway-management-extension.js
  11. 2
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  12. 59
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java
  13. 71
      application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java
  14. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java
  15. 74
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  16. 2
      application/src/main/resources/thingsboard.yml
  17. 4
      application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java
  18. 4
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java
  19. 3
      common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleService.java
  20. 12
      common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java
  21. 37
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
  22. 76
      dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java
  23. 1
      pom.xml

10
application/src/main/data/json/system/widget_types/gateway_configuration.json

@ -8,7 +8,15 @@
"type": "static",
"sizeX": 8,
"sizeY": 6.5,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-form\n [ctx]=\"ctx\">\n</tb-gateway-form>\n",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n}\n",

10
application/src/main/data/json/system/widget_types/gateway_configuration__single_device_.json

@ -8,7 +8,15 @@
"type": "latest",
"sizeX": 7.5,
"sizeY": 9,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-form\n [ctx]=\"ctx\"\n [isStateForm]=\"true\">\n</tb-gateway-form>",
"templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}",
"controllerScript": "self.onInit = function() {\n}\n\n\nself.onDestroy = function() {\n}\n\nself.typeParameters = function() {\n return {\n maxDatasources: 1,\t\t\t\n dataKeysOptional: true,\n singleEntity: true\n };\n}\n\n",

10
application/src/main/data/json/system/widget_types/gateway_connectors.json

@ -8,7 +8,15 @@
"type": "latest",
"sizeX": 11,
"sizeY": 8,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-connector [device]=\"entityId\" *ngIf=\"entityId\" [ctx]=\"ctx\"></tb-gateway-connector>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n if (self.ctx.datasources && self.ctx.datasources.length) {\n self.ctx.$scope.entityId = self.ctx.datasources[0].entity.id;\n }\n};\n\nself.onDataUpdated = function() {\n self.ctx.$scope.gatewayConnectors?.onDataUpdated();\n};\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true,\n singleEntity: true\n };\n}",

10
application/src/main/data/json/system/widget_types/gateway_custom_statistics.json

@ -8,7 +8,15 @@
"type": "timeseries",
"sizeX": 8,
"sizeY": 5,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-statistics [ctx]=ctx></tb-gateway-statistics>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() { \n};\n\nself.onDataUpdated = function() {\n};\n\nself.onLatestDataUpdated = function() {\n};\n\nself.onResize = function() {\n};\n\nself.onEditModeChanged = function() {\n};\n\nself.onDestroy = function() {\n};\n\nself.typeParameters = function() {\n return {\n hasAdditionalLatestDataKeys: false,\n dataKeysOptional: true\n };\n}\n",

10
application/src/main/data/json/system/widget_types/gateway_general_chart_statistics.json

@ -8,7 +8,15 @@
"type": "timeseries",
"sizeX": 8,
"sizeY": 5,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-statistics [ctx]=ctx [general]='true'></tb-gateway-statistics>",
"templateCss": ".legend {\n font-size: 13px;\n line-height: 10px;\n}\n\n.legend table { \n border-spacing: 0px;\n border-collapse: separate;\n}\n\n.mouse-events .flot-overlay {\n cursor: crosshair; \n}\n\n",
"controllerScript": "self.onInit = function() { \n};\n\nself.onDataUpdated = function() {\n};\n\nself.onLatestDataUpdated = function() {\n};\n\nself.onResize = function() {\n};\n\nself.onEditModeChanged = function() {\n};\n\nself.onDestroy = function() {\n};\n\nself.typeParameters = function() {\n return {\n hasAdditionalLatestDataKeys: false\n };\n}\n",

10
application/src/main/data/json/system/widget_types/gateway_general_configuration.json

@ -8,7 +8,15 @@
"type": "latest",
"sizeX": 11,
"sizeY": 8,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-configuration [device]=\"entityId\" *ngIf=\"entityId\"></tb-gateway-configuration>",
"templateCss": "",
"controllerScript": "self.onInit = function() {\n if (self.ctx.datasources && self.ctx.datasources.length) {\n self.ctx.$scope.entityId = self.ctx.datasources[0].entity.id;\n }\n};\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true,\n singleEntity: true\n };\n}",

10
application/src/main/data/json/system/widget_types/gateway_logs.json

@ -8,7 +8,15 @@
"type": "timeseries",
"sizeX": 7.5,
"sizeY": 3,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-logs [ctx]=\"ctx\">\n \n</tb-gateway-logs>",
"templateCss": "#container {\n overflow: auto;\n}\n\n.tbDatasource-container {\n margin: 5px;\n padding: 8px;\n}\n\n.tbDatasource-title {\n font-size: 1.200rem;\n font-weight: 500;\n padding-bottom: 10px;\n}\n\n.tbDatasource-table {\n width: 100%;\n box-shadow: 0 0 10px #ccc;\n border-collapse: collapse;\n white-space: nowrap;\n font-size: 1.000rem;\n color: #757575;\n}\n\n.tbDatasource-table td {\n position: relative;\n border-top: 1px solid rgba(0, 0, 0, 0.12);\n border-bottom: 1px solid rgba(0, 0, 0, 0.12);\n padding: 0px 18px;\n box-sizing: border-box;\n}",
"controllerScript": "self.onInit = function() {\n};\n\nself.typeParameters = function() {\n return {\n dataKeysOptional: true,\n singleEntity: true\n };\n}",

10
application/src/main/data/json/system/widget_types/service_rpc.json

@ -8,7 +8,15 @@
"type": "rpc",
"sizeX": 8.5,
"sizeY": 5.5,
"resources": [],
"resources": [
{
"url": {
"entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}"
},
"isModule": true
}
],
"templateHtml": "<tb-gateway-service-rpc [ctx]=\"ctx\"></tb-gateway-service-rpc>",
"templateCss": ".error {\n font-size: 14px !important;\n color: maroon;/*rgb(250,250,250);*/\n background-color: transparent;\n padding: 6px;\n}\n\n.error span {\n margin: auto;\n}\n\n.gpio-panel {\n padding-top: 10px;\n white-space: nowrap;\n}\n\n.gpio-panel section[fxflex] {\n min-width: 0px;\n}\n\n\n.switch-panel {\n margin: 0;\n height: 32px;\n width: 66px;\n min-width: 66px;\n}\n\n.switch-panel mat-slide-toggle {\n margin: 0;\n width: 36px;\n min-width: 36px;\n}\n\n.switch-panel.col-0 mat-slide-toggle {\n margin-left: 8px;\n margin-right: 4px;\n}\n\n.switch-panel.col-1 mat-slide-toggle {\n margin-left: 4px;\n margin-right: 8px;\n}\n\n.gpio-row {\n height: 32px;\n}\n\n.pin {\n margin-top: auto;\n margin-bottom: auto;\n color: white;\n font-size: 12px;\n width: 16px;\n min-width: 16px;\n}\n\n.switch-panel.col-0 .pin {\n margin-left: auto;\n padding-left: 2px;\n text-align: right;\n}\n\n.switch-panel.col-1 .pin {\n margin-right: auto;\n \n text-align: left;\n}\n\n.gpio-left-label {\n margin-right: 8px;\n}\n\n.gpio-right-label {\n margin-left: 8px;\n}",
"controllerScript": "\nself.onInit = function() {\n};",

7031
application/src/main/data/resources/dashboards/gateways_dashboard.json

File diff suppressed because it is too large

1
application/src/main/data/resources/js_modules/gateway-management-extension.js

File diff suppressed because one or more lines are too long

2
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -154,6 +154,7 @@ public class ThingsboardInstallService {
entityDatabaseSchemaService.createOrUpdateDeviceInfoView(persistToTelemetry);
log.info("Updating system data...");
dataUpdateService.upgradeRuleNodes();
installScripts.loadSystemResources();
systemDataLoaderService.loadSystemWidgets();
installScripts.loadSystemLwm2mResources();
installScripts.loadSystemImages();
@ -194,6 +195,7 @@ public class ThingsboardInstallService {
systemDataLoaderService.createDefaultTenantProfiles();
systemDataLoaderService.createAdminSettings();
systemDataLoaderService.createRandomJwtSettings();
installScripts.loadSystemResources();
systemDataLoaderService.loadSystemWidgets();
systemDataLoaderService.createOAuth2Templates();
systemDataLoaderService.createQueues();

59
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java

@ -15,26 +15,22 @@
*/
package org.thingsboard.server.service.entitiy.dashboard;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService;
import org.thingsboard.server.service.sync.GitSyncService;
import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@ -47,18 +43,19 @@ public class DashboardSyncService {
private final GitSyncService gitSyncService;
private final ResourceService resourceService;
private final TbWidgetsBundleService tbWidgetsBundleService;
private final WidgetsBundleService widgetsBundleService;
private final PartitionService partitionService;
@Value("${transport.gateway.dashboard.sync.enabled:true}")
private boolean enabled;
@Value("${transport.gateway.dashboard.sync.repository_url:}")
private String repoUrl;
@Value("${transport.gateway.dashboard.sync.branch:main}")
private String branch;
@Value("${transport.gateway.dashboard.sync.fetch_frequency:24}")
private int fetchFrequencyHours;
private static final String REPO_KEY = "gateways-dashboard";
private static final String GATEWAY_RESOURCE_ID_PARAM = "${GATEWAY_RESOURCE_ID}";
private static final String GATEWAYS_DASHBOARD_KEY = "gateways_dashboard.json";
@AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
@ -66,7 +63,7 @@ public class DashboardSyncService {
if (!enabled) {
return;
}
gitSyncService.registerSync(REPO_KEY, repoUrl, "main", TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update);
gitSyncService.registerSync(REPO_KEY, repoUrl, branch, TimeUnit.HOURS.toMillis(fetchFrequencyHours), this::update);
}
private void update() {
@ -74,47 +71,25 @@ public class DashboardSyncService {
return;
}
RepoFile extensionResourceFile = listFiles("resources").get(0);
String data = getFileContent(extensionResourceFile.path());
TbResource extensionResource = createOrUpdateResource(ResourceType.JS_MODULE, extensionResourceFile.name(), data.getBytes(StandardCharsets.UTF_8));
String extensionResourceId = extensionResource.getUuidId().toString();
List<RepoFile> resources = listFiles("resources");
for (RepoFile resourceFile : resources) {
String data = getFileContent(resourceFile.path());
resourceService.updateSystemResource(ResourceType.JS_MODULE, resourceFile.name(), data);
}
Stream<JsonNode> widgetsBundles = listFiles("widget_bundles").stream()
.map(widgetsBundleFile -> {
String widgetsBundleDescriptor = getFileContent(widgetsBundleFile.path());
widgetsBundleDescriptor = widgetsBundleDescriptor.replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId);
return JacksonUtil.toJsonNode(widgetsBundleDescriptor);
});
Stream<JsonNode> widgetTypes = listFiles("widget_types").stream()
.map(widgetTypeFile -> {
String widgetTypeDetails = getFileContent(widgetTypeFile.path());
widgetTypeDetails = widgetTypeDetails.replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId);
return JacksonUtil.toJsonNode(widgetTypeDetails);
});
tbWidgetsBundleService.updateWidgets(TenantId.SYS_TENANT_ID, widgetsBundles, widgetTypes);
Stream<String> widgetsBundles = listFiles("widget_bundles").stream()
.map(widgetsBundleFile -> getFileContent(widgetsBundleFile.path()));
Stream<String> widgetTypes = listFiles("widget_types").stream()
.map(widgetTypeFile -> getFileContent(widgetTypeFile.path()));
widgetsBundleService.updateSystemWidgets(widgetsBundles, widgetTypes);
RepoFile dashboardFile = listFiles("dashboards").get(0);
String dashboardJson = getFileContent(dashboardFile.path()).replace(GATEWAY_RESOURCE_ID_PARAM, extensionResourceId);
createOrUpdateResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, dashboardJson.getBytes(StandardCharsets.UTF_8));
String dashboardJson = getFileContent(dashboardFile.path());
resourceService.updateSystemResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, dashboardJson);
log.info("Gateways dashboard sync completed");
}
private TbResource createOrUpdateResource(ResourceType resourceType, String resourceKey, byte[] data) {
TbResource resource = resourceService.findResourceByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey);
if (resource == null) {
resource = new TbResource();
resource.setTenantId(TenantId.SYS_TENANT_ID);
resource.setResourceType(resourceType);
resource.setResourceKey(resourceKey);
resource.setFileName(resourceKey);
resource.setTitle(resourceKey);
}
resource.setData(data);
log.debug("{} resource {}", (resource.getId() == null ? "Creating" : "Updating"), resourceKey);
return resourceService.saveResource(resource);
}
private List<RepoFile> listFiles(String path) {
return gitSyncService.listFiles(REPO_KEY, path, 1, FileType.FILE);
}

71
application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/DefaultWidgetsBundleService.java

@ -15,29 +15,22 @@
*/
package org.thingsboard.server.service.entitiy.widgets.bundle;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Stream;
@Service
@TbCoreComponent
@ -89,68 +82,4 @@ public class DefaultWidgetsBundleService extends AbstractTbEntityService impleme
autoCommit(user, widgetsBundleId);
}
@Transactional
@Override
public void updateWidgets(TenantId tenantId, Stream<JsonNode> bundles, Stream<JsonNode> widgets) {
widgets.forEach(widgetTypeJson -> {
try {
WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class);
WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(tenantId, widgetTypeDetails.getFqn());
if (existingWidget != null) {
widgetTypeDetails.setId(existingWidget.getId());
widgetTypeDetails.setCreatedTime(existingWidget.getCreatedTime());
}
widgetTypeDetails.setTenantId(tenantId);
widgetTypeService.saveWidgetType(widgetTypeDetails);
log.debug("{} widget type {}", existingWidget == null ? "Created" : "Updated", widgetTypeDetails.getFqn());
} catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetTypeJson, e);
}
});
bundles.forEach(widgetsBundleDescriptorJson -> {
if (widgetsBundleDescriptorJson == null || !widgetsBundleDescriptorJson.has("widgetsBundle")) {
throw new RuntimeException("Invalid widgets bundle json: [" + widgetsBundleDescriptorJson + "]");
}
JsonNode widgetsBundleJson = widgetsBundleDescriptorJson.get("widgetsBundle");
WidgetsBundle widgetsBundle = JacksonUtil.treeToValue(widgetsBundleJson, WidgetsBundle.class);
WidgetsBundle existingWidgetsBundle = widgetsBundleService.findWidgetsBundleByTenantIdAndAlias(tenantId, widgetsBundle.getAlias());
if (existingWidgetsBundle != null) {
widgetsBundle.setId(existingWidgetsBundle.getId());
widgetsBundle.setCreatedTime(existingWidgetsBundle.getCreatedTime());
}
widgetsBundle.setTenantId(tenantId);
widgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle);
log.debug("{} widgets bundle {}", existingWidgetsBundle == null ? "Created" : "Updated", widgetsBundle.getAlias());
List<String> widgetTypeFqns = new ArrayList<>();
if (widgetsBundleDescriptorJson.has("widgetTypes")) {
JsonNode widgetTypesArrayJson = widgetsBundleDescriptorJson.get("widgetTypes");
widgetTypesArrayJson.forEach(widgetTypeJson -> {
try {
WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class);
WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(tenantId, widgetTypeDetails.getFqn());
if (existingWidget != null) {
widgetTypeDetails.setId(existingWidget.getId());
widgetTypeDetails.setCreatedTime(existingWidget.getCreatedTime());
}
widgetTypeDetails.setTenantId(tenantId);
widgetTypeDetails = widgetTypeService.saveWidgetType(widgetTypeDetails);
widgetTypeFqns.add(widgetTypeDetails.getFqn());
} catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetsBundleDescriptorJson, e);
}
});
}
if (widgetsBundleDescriptorJson.has("widgetTypeFqns")) {
JsonNode widgetFqnsArrayJson = widgetsBundleDescriptorJson.get("widgetTypeFqns");
widgetFqnsArrayJson.forEach(fqnJson -> {
widgetTypeFqns.add(fqnJson.asText());
});
}
widgetTypeService.updateWidgetsBundleWidgetFqns(tenantId, widgetsBundle.getId(), widgetTypeFqns);
});
}
}

4
application/src/main/java/org/thingsboard/server/service/entitiy/widgets/bundle/TbWidgetsBundleService.java

@ -15,16 +15,13 @@
*/
package org.thingsboard.server.service.entitiy.widgets.bundle;
import com.fasterxml.jackson.databind.JsonNode;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.service.entitiy.SimpleTbEntityService;
import java.util.List;
import java.util.stream.Stream;
public interface TbWidgetsBundleService extends SimpleTbEntityService<WidgetsBundle> {
@ -32,6 +29,5 @@ public interface TbWidgetsBundleService extends SimpleTbEntityService<WidgetsBun
void updateWidgetsBundleWidgetFqns(WidgetsBundleId widgetsBundleId, List<String> widgetFqns, User user) throws Exception;
void updateWidgets(TenantId tenantId, Stream<JsonNode> bundles, Stream<JsonNode> widgets);
}

74
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -17,7 +17,6 @@ package org.thingsboard.server.service.install;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Streams;
import lombok.Getter;
import lombok.Setter;
import lombok.SneakyThrows;
@ -49,10 +48,11 @@ import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.util.ImageUtils;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.install.update.ImagesUpdater;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@ -89,7 +89,7 @@ public class InstallScripts {
public static final String OAUTH2_CONFIG_TEMPLATES_DIR = "oauth2_config_templates";
public static final String DASHBOARDS_DIR = "dashboards";
public static final String MODELS_LWM2M_DIR = "lwm2m-registry";
public static final String CREDENTIALS_DIR = "credentials";
public static final String RESOURCES_DIR = "resources";
public static final String JSON_EXT = ".json";
public static final String SVG_EXT = ".svg";
@ -108,7 +108,7 @@ public class InstallScripts {
private WidgetTypeService widgetTypeService;
@Autowired
private TbWidgetsBundleService tbWidgetsBundleService;
private WidgetsBundleService widgetsBundleService;
@Autowired
private OAuth2ConfigTemplateService oAuth2TemplateService;
@ -212,35 +212,10 @@ public class InstallScripts {
log.info("Loading system widgets");
Path widgetBundlesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_BUNDLES_DIR);
Stream<JsonNode> bundles;
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(widgetBundlesDir, path -> path.toString().endsWith(JSON_EXT))) {
bundles = Streams.stream(dirStream).map(path -> {
try {
return JacksonUtil.toJsonNode(path.toFile());
} catch (Exception e) {
log.error("Unable to parse widgets bundle from json: [{}]", path);
throw new RuntimeException("Unable to parse widgets bundle from json", e);
}
});
}
Stream<JsonNode> widgets;
Stream<String> bundles = listDir(widgetBundlesDir).filter(path -> path.toString().endsWith(JSON_EXT)).map(this::getContent);
Path widgetTypesDir = Paths.get(getDataDir(), JSON_DIR, SYSTEM_DIR, WIDGET_TYPES_DIR);
if (Files.exists(widgetTypesDir)) {
try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(widgetTypesDir, path -> path.toString().endsWith(JSON_EXT))) {
widgets = Streams.stream(dirStream).map(path -> {
try {
return JacksonUtil.toJsonNode(path.toFile());
} catch (Exception e) {
log.error("Unable to parse widget type from json: [{}]", path);
throw new RuntimeException("Unable to parse widget type from json", e);
}
});
}
} else {
widgets = Stream.empty();
}
tbWidgetsBundleService.updateWidgets(TenantId.SYS_TENANT_ID, bundles, widgets);
Stream<String> widgets = listDir(widgetTypesDir).filter(path -> path.toString().endsWith(JSON_EXT)).map(this::getContent);
widgetsBundleService.updateSystemWidgets(bundles, widgets);
loadSystemScadaSymbols();
}
@ -449,6 +424,41 @@ public class InstallScripts {
}
}
public void loadSystemResources() {
Path resourcesDir = Path.of(getDataDir(), RESOURCES_DIR);
loadSystemResources(resourcesDir.resolve("js_modules"), ResourceType.JS_MODULE);
loadSystemResources(resourcesDir.resolve("dashboards"), ResourceType.DASHBOARD);
}
private void loadSystemResources(Path dir, ResourceType resourceType) {
listDir(dir).forEach(resourceFile -> {
String resourceKey = resourceFile.getFileName().toString();
try {
String data = getContent(resourceFile);
TbResource resource = resourceService.updateSystemResource(resourceType, resourceKey, data);
log.info("{} resource {}", (resource.getId() == null ? "Created" : "Updated"), resourceKey);
} catch (Exception e) {
throw new RuntimeException("Unable to load system resource " + resourceFile, e);
}
});
}
private String getContent(Path file) {
try {
return Files.readString(file);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
private Stream<Path> listDir(Path resourcesDir) {
try {
return Files.list(resourcesDir);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
private void doSaveLwm2mResource(TbResource resource) throws ThingsboardException {
log.trace("Executing saveResource [{}]", resource);
if (resource.getData() == null || resource.getData().length == 0) {

2
application/src/main/resources/thingsboard.yml

@ -1251,6 +1251,8 @@ transport:
enabled: "${TB_GATEWAY_DASHBOARD_SYNC_ENABLED:true}"
# URL of gateways dashboard repository
repository_url: "${TB_GATEWAY_DASHBOARD_SYNC_REPOSITORY_URL:https://github.com/thingsboard/gateway-management-extensions-dist.git}"
# Branch of gateways dashboard repository to work with
branch: "${TB_GATEWAY_DASHBOARD_SYNC_BRANCH:main}"
# Fetch frequency in hours for gateways dashboard repository
fetch_frequency: "${TB_GATEWAY_DASHBOARD_SYNC_FETCH_FREQUENCY:24}"

4
application/src/test/java/org/thingsboard/server/service/install/InstallScriptsTest.java

@ -36,7 +36,7 @@ import org.thingsboard.server.dao.service.validator.RuleChainDataValidator;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiLimitService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.service.entitiy.widgets.bundle.TbWidgetsBundleService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.install.update.ImagesUpdater;
import java.io.IOException;
@ -60,7 +60,7 @@ class InstallScriptsTest {
@MockBean
WidgetTypeService widgetTypeService;
@MockBean
TbWidgetsBundleService tbWidgetsBundleService;
WidgetsBundleService widgetsBundleService;
@MockBean
OAuth2ConfigTemplateService oAuth2TemplateService;
@MockBean

4
common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java

@ -64,4 +64,8 @@ public interface ResourceService extends EntityDaoService {
long sumDataSizeByTenantId(TenantId tenantId);
TbResource updateSystemResource(ResourceType resourceType, String resourceKey, String data);
String checkSystemResourcesUsage(String content, ResourceType... usedResourceTypes);
}

3
common/dao-api/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleService.java

@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.widget.WidgetsBundleFilter;
import org.thingsboard.server.dao.entity.EntityDaoService;
import java.util.List;
import java.util.stream.Stream;
public interface WidgetsBundleService extends EntityDaoService {
@ -49,4 +50,6 @@ public interface WidgetsBundleService extends EntityDaoService {
void deleteWidgetsBundlesByTenantId(TenantId tenantId);
void updateSystemWidgets(Stream<String> bundles, Stream<String> widgets);
}

12
common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java

@ -17,16 +17,24 @@ package org.thingsboard.common.util;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.intellij.lang.annotations.Language;
import org.springframework.util.ConcurrentReferenceHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Function;
import java.util.function.UnaryOperator;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.springframework.util.ConcurrentReferenceHashMap.ReferenceType.SOFT;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class RegexUtils {
public static final Pattern UUID_PATTERN = Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}");
private static final ConcurrentMap<String, Pattern> patternsCache = new ConcurrentReferenceHashMap<>(16, SOFT);
public static String replace(String s, Pattern pattern, UnaryOperator<String> replacer) {
return pattern.matcher(s).replaceAll(matchResult -> {
@ -34,6 +42,10 @@ public class RegexUtils {
});
}
public static String replace(String input, @Language("regexp") String pattern, Function<MatchResult, String> replacer) {
return patternsCache.computeIfAbsent(pattern, Pattern::compile).matcher(input).replaceAll(replacer);
}
public static boolean matches(String input, Pattern pattern) {
return pattern.matcher(input).matches();
}

37
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@ -23,6 +23,7 @@ import org.hibernate.exception.ConstraintViolationException;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.common.util.RegexUtils;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent;
import org.thingsboard.server.common.data.EntityType;
@ -44,6 +45,7 @@ import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
@ -237,6 +239,41 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return resourceDao.sumDataSizeByTenantId(tenantId);
}
@Override
public TbResource updateSystemResource(ResourceType resourceType, String resourceKey, String data) {
if (resourceType == ResourceType.DASHBOARD) {
data = checkSystemResourcesUsage(data, ResourceType.JS_MODULE);
}
TbResource resource = findResourceByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey);
if (resource == null) {
resource = new TbResource();
resource.setTenantId(TenantId.SYS_TENANT_ID);
resource.setResourceType(resourceType);
resource.setResourceKey(resourceKey);
resource.setFileName(resourceKey);
resource.setTitle(resourceKey);
}
resource.setData(data.getBytes(StandardCharsets.UTF_8));
log.debug("{} system resource {}", (resource.getId() == null ? "Creating" : "Updating"), resourceKey);
return saveResource(resource);
}
@Override
public String checkSystemResourcesUsage(String content, ResourceType... usedResourceTypes) {
return RegexUtils.replace(content, "\\$\\{RESOURCE:(.+)}", matchResult -> {
String resourceKey = matchResult.group(1);
for (ResourceType resourceType : usedResourceTypes) {
TbResourceInfo resource = findResourceInfoByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey);
if (resource != null) {
log.trace("Replaced '{}' with resource id {}", matchResult.group(), resource.getUuidId());
return resource.getUuidId().toString();
}
}
return "";
});
}
protected String calculateEtag(byte[] data) {
return Hashing.sha256().hashBytes(data).toString();
}

76
dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java

@ -15,17 +15,23 @@
*/
package org.thingsboard.server.dao.widget;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.widget.WidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.common.data.widget.WidgetsBundleFilter;
import org.thingsboard.server.dao.entity.AbstractCachedEntityService;
@ -33,6 +39,7 @@ import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.service.Validator;
@ -40,6 +47,7 @@ import org.thingsboard.server.dao.service.Validator;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
@Service("WidgetsBundleDaoService")
@Slf4j
@ -64,6 +72,9 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService {
@Autowired
private ImageService imageService;
@Autowired
private ResourceService resourceService;
@Override
public WidgetsBundle findWidgetsBundleById(TenantId tenantId, WidgetsBundleId widgetsBundleId) {
log.trace("Executing findWidgetsBundleById [{}]", widgetsBundleId);
@ -196,6 +207,71 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService {
deleteWidgetsBundlesByTenantId(tenantId);
}
@Transactional
@Override
public void updateSystemWidgets(Stream<String> bundles, Stream<String> widgets) {
widgets.forEach(widgetTypeJson -> {
try {
updateSystemWidget(widgetTypeJson);
} catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetTypeJson, e);
}
});
bundles.forEach(widgetsBundleDescriptorJson -> {
JsonNode widgetsBundleDescriptor = JacksonUtil.toJsonNode(widgetsBundleDescriptorJson);
if (widgetsBundleDescriptor == null || !widgetsBundleDescriptor.has("widgetsBundle")) {
throw new RuntimeException("Invalid widgets bundle json: [" + widgetsBundleDescriptorJson + "]");
}
JsonNode widgetsBundleJson = widgetsBundleDescriptor.get("widgetsBundle");
WidgetsBundle widgetsBundle = JacksonUtil.treeToValue(widgetsBundleJson, WidgetsBundle.class);
WidgetsBundle existingWidgetsBundle = findWidgetsBundleByTenantIdAndAlias(TenantId.SYS_TENANT_ID, widgetsBundle.getAlias());
if (existingWidgetsBundle != null) {
widgetsBundle.setId(existingWidgetsBundle.getId());
widgetsBundle.setCreatedTime(existingWidgetsBundle.getCreatedTime());
}
widgetsBundle.setTenantId(TenantId.SYS_TENANT_ID);
widgetsBundle = saveWidgetsBundle(widgetsBundle);
log.debug("{} widgets bundle {}", existingWidgetsBundle == null ? "Created" : "Updated", widgetsBundle.getAlias());
List<String> widgetTypeFqns = new ArrayList<>();
if (widgetsBundleDescriptor.has("widgetTypes")) {
JsonNode widgetTypesArrayJson = widgetsBundleDescriptor.get("widgetTypes");
widgetTypesArrayJson.forEach(widgetTypeJson -> {
try {
WidgetTypeDetails widgetTypeDetails = updateSystemWidget(widgetTypeJson.toString());
widgetTypeFqns.add(widgetTypeDetails.getFqn());
} catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetsBundleDescriptorJson, e);
}
});
}
if (widgetsBundleDescriptor.has("widgetTypeFqns")) {
JsonNode widgetFqnsArrayJson = widgetsBundleDescriptor.get("widgetTypeFqns");
widgetFqnsArrayJson.forEach(fqnJson -> {
widgetTypeFqns.add(fqnJson.asText());
});
}
widgetTypeService.updateWidgetsBundleWidgetFqns(TenantId.SYS_TENANT_ID, widgetsBundle.getId(), widgetTypeFqns);
});
}
private WidgetTypeDetails updateSystemWidget(String widgetTypeJson) {
widgetTypeJson = resourceService.checkSystemResourcesUsage(widgetTypeJson, ResourceType.JS_MODULE);
WidgetTypeDetails widgetTypeDetails = JacksonUtil.fromString(widgetTypeJson, WidgetTypeDetails.class);
WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(TenantId.SYS_TENANT_ID, widgetTypeDetails.getFqn());
if (existingWidget != null) {
widgetTypeDetails.setId(existingWidget.getId());
widgetTypeDetails.setCreatedTime(existingWidget.getCreatedTime());
}
widgetTypeDetails.setTenantId(TenantId.SYS_TENANT_ID);
widgetTypeDetails = widgetTypeService.saveWidgetType(widgetTypeDetails);
log.debug("{} widget type {}", existingWidget == null ? "Created" : "Updated", widgetTypeDetails.getFqn());
return widgetTypeDetails;
}
@Override
public Optional<HasId<?>> findEntity(TenantId tenantId, EntityId entityId) {
return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId())));

1
pom.xml

@ -849,6 +849,7 @@
<exclude>.run/**</exclude>
<exclude>**/NetworkReceive.java</exclude>
<exclude>**/lwm2m-registry/**</exclude>
<exclude>src/main/data/resources/**</exclude>
</excludes>
<mapping>
<proto>JAVADOC_STYLE</proto>

Loading…
Cancel
Save