Browse Source

Use links to reference resources

pull/11873/head
ViacheslavKlimov 2 years ago
parent
commit
667d32d8aa
  1. 2
      application/src/main/data/json/system/widget_types/gateway_configuration.json
  2. 2
      application/src/main/data/json/system/widget_types/gateway_configuration__single_device_.json
  3. 2
      application/src/main/data/json/system/widget_types/gateway_connectors.json
  4. 2
      application/src/main/data/json/system/widget_types/gateway_custom_statistics.json
  5. 2
      application/src/main/data/json/system/widget_types/gateway_general_chart_statistics.json
  6. 2
      application/src/main/data/json/system/widget_types/gateway_general_configuration.json
  7. 2
      application/src/main/data/json/system/widget_types/gateway_logs.json
  8. 2
      application/src/main/data/json/system/widget_types/service_rpc.json
  9. 2
      application/src/main/data/resources/dashboards/gateways_dashboard.json
  10. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  11. 6
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  12. 59
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  13. 4
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
  15. 12
      application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java
  16. 25
      application/src/test/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncServiceTest.java
  17. 32
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  18. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java
  19. 32
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
  20. 11
      dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java

2
application/src/main/data/json/system/widget_types/gateway_configuration.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_configuration__single_device_.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_connectors.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_custom_statistics.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_general_chart_statistics.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_general_configuration.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/gateway_logs.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/json/system/widget_types/service_rpc.json

@ -12,7 +12,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

2
application/src/main/data/resources/dashboards/gateways_dashboard.json

@ -1923,7 +1923,7 @@
{ {
"url": { "url": {
"entityType": "TB_RESOURCE", "entityType": "TB_RESOURCE",
"id": "${RESOURCE:gateway-management-extension.js}" "id": "tb-resource;/api/resource/js_module/system/gateway-management-extension.js"
}, },
"isModule": true "isModule": true
} }

4
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -35,7 +35,6 @@ import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
@ -72,8 +71,6 @@ public abstract class AbstractTbEntityService {
@Autowired(required = false) @Autowired(required = false)
@Lazy @Lazy
private EntitiesVersionControlService vcService; private EntitiesVersionControlService vcService;
@Autowired
protected AccessControlService accessControlService;
protected boolean isTestProfile() { protected boolean isTestProfile() {
return Set.of(this.env.getActiveProfiles()).contains("test"); return Set.of(this.env.getActiveProfiles()).contains("test");
@ -123,4 +120,5 @@ public abstract class AbstractTbEntityService {
return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!"));
} }
} }
} }

6
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -247,10 +247,8 @@ public class InstallScripts {
dirStream.forEach( dirStream.forEach(
path -> { path -> {
try { try {
String widgetTypeJson = Files.readString(path); JsonNode widgetTypeJson = JacksonUtil.toJsonNode(path.toFile());
widgetTypeJson = resourceService.checkSystemResourcesUsage(widgetTypeJson, ResourceType.JS_MODULE); WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class);
WidgetTypeDetails widgetTypeDetails = JacksonUtil.fromString(widgetTypeJson, WidgetTypeDetails.class);
widgetTypeService.saveWidgetType(widgetTypeDetails); widgetTypeService.saveWidgetType(widgetTypeDetails);
} catch (Exception e) { } catch (Exception e) {
log.error("Unable to load widget type from json: [{}]", path.toString()); log.error("Unable to load widget type from json: [{}]", path.toString());

59
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -27,19 +27,26 @@ import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.alarm.AlarmSeverity;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.DynamicValue;
import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration; import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.sql.JpaExecutorService; import org.thingsboard.server.dao.sql.JpaExecutorService;
import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.component.RuleNodeClassInfo; import org.thingsboard.server.service.component.RuleNodeClassInfo;
import org.thingsboard.server.utils.TbNodeUpgradeUtils; import org.thingsboard.server.utils.TbNodeUpgradeUtils;
@ -81,6 +88,15 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired @Autowired
private TenantProfileService tenantProfileService; private TenantProfileService tenantProfileService;
@Autowired
private ResourceService resourceService;
@Autowired
private DashboardService dashboardService;
@Autowired
private WidgetTypeService widgetTypeService;
@Override @Override
public void updateData(String fromVersion) throws Exception { public void updateData(String fromVersion) throws Exception {
switch (fromVersion) { switch (fromVersion) {
@ -94,6 +110,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
updateMaxRuleNodeExecsPerMessage(); updateMaxRuleNodeExecsPerMessage();
updateGatewayRateLimits(); updateGatewayRateLimits();
break; break;
case "3.8.1":
updateResourcesUsage();
break;
default: default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
} }
@ -341,6 +360,46 @@ public class DefaultDataUpdateService implements DataUpdateService {
return false; return false;
} }
private void updateResourcesUsage() {
log.info("Updating resources usage in dashboards");
var dashboards = new PageDataIterable<>(dashboardService::findAllDashboardsIds, 512);
int totalCount = 0;
int updatedCount = 0;
for (DashboardId dashboardId : dashboards) {
Dashboard dashboard = dashboardService.findDashboardById(TenantId.SYS_TENANT_ID, dashboardId);
boolean updated = resourceService.replaceResourcesUsageWithUrls(dashboard);
if (updated) {
dashboardService.saveDashboard(dashboard);
updatedCount++;
}
totalCount++;
if (totalCount % 1000 == 0) {
log.info("Processed {} dashboards, updated {}", totalCount, updatedCount);
}
}
log.info("Updated {} dashboards", updatedCount);
log.info("Updating resources usage in widgets");
totalCount = 0;
updatedCount = 0;
var widgets = new PageDataIterable<>(widgetTypeService::findAllWidgetTypesIds, 512);
for (WidgetTypeId widgetTypeId : widgets) {
WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(TenantId.SYS_TENANT_ID, widgetTypeId);
boolean updated = resourceService.replaceResourcesUsageWithUrls(widgetTypeDetails);
if (updated) {
widgetTypeService.saveWidgetType(widgetTypeDetails);
updatedCount++;
}
totalCount++;
if (totalCount % 200 == 0) {
log.info("Processed {} widgets, updated {}", totalCount, updatedCount);
}
}
log.info("Updated {} widgets", updatedCount);
}
public static boolean getEnv(String name, boolean defaultValue) { public static boolean getEnv(String name, boolean defaultValue) {
String env = System.getenv(name); String env = System.getenv(name);
if (env == null) { if (env == null) {

4
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java

@ -40,6 +40,7 @@ import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.permission.Resource;
@ -59,13 +60,16 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
private final TbClusterService clusterService; private final TbClusterService clusterService;
private final ImageService imageService; private final ImageService imageService;
private final AccessControlService accessControlService;
private final Cache<ImageCacheKey, String> etagCache; private final Cache<ImageCacheKey, String> etagCache;
public DefaultTbImageService(TbClusterService clusterService, ImageService imageService, public DefaultTbImageService(TbClusterService clusterService, ImageService imageService,
AccessControlService accessControlService,
@Value("${cache.image.etag.timeToLiveInMinutes:44640}") int cacheTtl, @Value("${cache.image.etag.timeToLiveInMinutes:44640}") int cacheTtl,
@Value("${cache.image.etag.maxSize:10000}") int cacheMaxSize) { @Value("${cache.image.etag.maxSize:10000}") int cacheMaxSize) {
this.clusterService = clusterService; this.clusterService = clusterService;
this.imageService = imageService; this.imageService = imageService;
this.accessControlService = accessControlService;
this.etagCache = Caffeine.newBuilder() this.etagCache = Caffeine.newBuilder()
.expireAfterAccess(cacheTtl, TimeUnit.MINUTES) .expireAfterAccess(cacheTtl, TimeUnit.MINUTES)
.maximumSize(cacheMaxSize) .maximumSize(cacheMaxSize)

2
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@ -39,6 +39,7 @@ import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.permission.Resource;
@ -66,6 +67,7 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
private final ResourceService resourceService; private final ResourceService resourceService;
private final ImageService imageService; private final ImageService imageService;
private final TbImageService tbImageService; private final TbImageService tbImageService;
private final AccessControlService accessControlService;
@Override @Override
public TbResource save(TbResource resource, SecurityUser user) throws ThingsboardException { public TbResource save(TbResource resource, SecurityUser user) throws ThingsboardException {

12
application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java

@ -200,7 +200,7 @@ public class ImageControllerTest extends AbstractControllerTest {
assertThat(exportData.getMediaType()).isEqualTo("image/png"); assertThat(exportData.getMediaType()).isEqualTo("image/png");
assertThat(exportData.getFileName()).isEqualTo(filename); assertThat(exportData.getFileName()).isEqualTo(filename);
assertThat(exportData.getTitle()).isEqualTo(filename); assertThat(exportData.getTitle()).isEqualTo(filename);
assertThat(exportData.getSubType()).isEqualTo(ResourceSubType.IMAGE.name()); assertThat(exportData.getSubType()).isEqualTo(ResourceSubType.IMAGE);
assertThat(exportData.getResourceKey()).isEqualTo(filename); assertThat(exportData.getResourceKey()).isEqualTo(filename);
assertThat(exportData.getData()).isEqualTo(Base64.getEncoder().encodeToString(PNG_IMAGE)); assertThat(exportData.getData()).isEqualTo(Base64.getEncoder().encodeToString(PNG_IMAGE));
assertThat(exportData.isPublic()).isTrue(); assertThat(exportData.isPublic()).isTrue();
@ -210,7 +210,7 @@ public class ImageControllerTest extends AbstractControllerTest {
TbResourceInfo importedImageInfo = doPut("/api/image/import", exportData, TbResourceInfo.class); TbResourceInfo importedImageInfo = doPut("/api/image/import", exportData, TbResourceInfo.class);
assertThat(importedImageInfo.getTitle()).isEqualTo(filename); assertThat(importedImageInfo.getTitle()).isEqualTo(filename);
assertThat(exportData.getSubType()).isEqualTo(ResourceSubType.IMAGE.name()); assertThat(exportData.getSubType()).isEqualTo(ResourceSubType.IMAGE);
assertThat(importedImageInfo.getResourceKey()).isEqualTo(filename); assertThat(importedImageInfo.getResourceKey()).isEqualTo(filename);
assertThat(importedImageInfo.getFileName()).isEqualTo(filename); assertThat(importedImageInfo.getFileName()).isEqualTo(filename);
assertThat(importedImageInfo.isPublic()).isTrue(); assertThat(importedImageInfo.isPublic()).isTrue();
@ -233,14 +233,14 @@ public class ImageControllerTest extends AbstractControllerTest {
String systemImageName = "my_system_png_image.png"; String systemImageName = "my_system_png_image.png";
TbResourceInfo systemImage = uploadImage(HttpMethod.POST, "/api/image", systemImageName, "image/png", PNG_IMAGE); TbResourceInfo systemImage = uploadImage(HttpMethod.POST, "/api/image", systemImageName, "image/png", PNG_IMAGE);
String systemScadaSymbolName = "my_system_scada_symbol_image.svg"; String systemScadaSymbolName = "my_system_scada_symbol_image.svg";
TbResourceInfo systemScadaSymbol = uploadImage(HttpMethod.POST, "/api/image", ResourceSubType.SCADA_SYMBOL.name(), systemScadaSymbolName, "image/svg+xml", SVG_IMAGE); TbResourceInfo systemScadaSymbol = uploadImage(HttpMethod.POST, "/api/image", ResourceSubType.SCADA_SYMBOL.name(), systemScadaSymbolName, "image/svg+xml", SVG_IMAGE);
loginTenantAdmin(); loginTenantAdmin();
String tenantImageName = "my_jpeg_image.jpg"; String tenantImageName = "my_jpeg_image.jpg";
TbResourceInfo tenantImage = uploadImage(HttpMethod.POST, "/api/image", tenantImageName, "image/jpeg", JPEG_IMAGE); TbResourceInfo tenantImage = uploadImage(HttpMethod.POST, "/api/image", tenantImageName, "image/jpeg", JPEG_IMAGE);
String tenantScadaSymbolName = "my_scada_symbol_image.svg"; String tenantScadaSymbolName = "my_scada_symbol_image.svg";
TbResourceInfo tenantScadaSymbol = uploadImage(HttpMethod.POST, "/api/image", ResourceSubType.SCADA_SYMBOL.name(), tenantScadaSymbolName, "image/svg+xml", SVG_IMAGE); TbResourceInfo tenantScadaSymbol = uploadImage(HttpMethod.POST, "/api/image", ResourceSubType.SCADA_SYMBOL.name(), tenantScadaSymbolName, "image/svg+xml", SVG_IMAGE);
List<TbResourceInfo> tenantImages = getImages(null, false, 10); List<TbResourceInfo> tenantImages = getImages(null, false, 10);
@ -262,7 +262,7 @@ public class ImageControllerTest extends AbstractControllerTest {
assertThat(getImages("my_system_scada_symbol", ResourceSubType.SCADA_SYMBOL.name(), true, 10)) assertThat(getImages("my_system_scada_symbol", ResourceSubType.SCADA_SYMBOL.name(), true, 10))
.containsOnly(systemScadaSymbol); .containsOnly(systemScadaSymbol);
assertThat(getImages("my_scada_symbol", ResourceSubType.SCADA_SYMBOL.name(),true, 10)) assertThat(getImages("my_scada_symbol", ResourceSubType.SCADA_SYMBOL.name(), true, 10))
.containsOnly(tenantScadaSymbol); .containsOnly(tenantScadaSymbol);
} }
@ -370,7 +370,7 @@ public class ImageControllerTest extends AbstractControllerTest {
private List<TbResourceInfo> getImages(String searchText, String imageSubType, boolean includeSystemImages, int limit) throws Exception { private List<TbResourceInfo> getImages(String searchText, String imageSubType, boolean includeSystemImages, int limit) throws Exception {
var url = "/api/images?includeSystemImages=" + includeSystemImages + "&"; var url = "/api/images?includeSystemImages=" + includeSystemImages + "&";
if (StringUtils.isNotEmpty(imageSubType)) { if (StringUtils.isNotEmpty(imageSubType)) {
url += "imageSubType=" + imageSubType+ "&"; url += "imageSubType=" + imageSubType + "&";
} }
PageData<TbResourceInfo> images = doGetTypedWithPageLink(url, new TypeReference<>() {}, new PageLink(limit, 0, searchText)); PageData<TbResourceInfo> images = doGetTypedWithPageLink(url, new TypeReference<>() {}, new PageLink(limit, 0, searchText));
return images.getData(); return images.getData();

25
application/src/test/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncServiceTest.java

@ -54,20 +54,39 @@ public class DashboardSyncServiceTest extends AbstractControllerTest {
resourceRepository.deleteAll(); resourceRepository.deleteAll();
} }
/*
* TODO - DISCUSS
* we can't use etag in the image/resource tag (image and resource can be updated) (OR CAN? - on export we convert urls to tags automatically.
* if don't want to update all resource's usages - just use link)
* but also cannot use key - it can be changed if resource/image with such key already exists.
* store resources alongside all system widgets/dashboards - same structure when exporting.
* for gateways dashboard repository - use link instead of tag, not to update ref each time the resource is updated.
*
*
* TODO:
* update system widgets with the new structure
* */
// FIXME: need to update resource key all the time???? because etag is changed. same for images....
// @Ignore
// TODO: update gateway dashboard repository; for now the test will fail because gateway-management-extension.js is referenced by id instead of tb-resource
@Test @Test
public void testGatewaysDashboardSync() throws Exception { public void testGatewaysDashboardSync() throws Exception {
loginTenantAdmin(); loginTenantAdmin();
await().atMost(60, TimeUnit.SECONDS).untilAsserted(() -> { await().atMost(45, TimeUnit.SECONDS).untilAsserted(() -> {
MockHttpServletResponse response = doGet("/api/resource/dashboard/system/gateways_dashboard.json") MockHttpServletResponse response = doGet("/api/resource/dashboard/system/gateways_dashboard.json")
.andExpect(status().isOk()) .andExpect(status().isOk())
.andReturn().getResponse(); .andReturn().getResponse();
String dashboardJson = response.getContentAsString(); String dashboardJson = response.getContentAsString();
String etag = response.getHeader("ETag");
assertThat(dashboardJson).contains("tb-resource;/api/resource/js_module/system/gateway-management-extension.js"); // checking that resource link is used
assertThat(dashboardJson).doesNotContain("${RESOURCE");
Dashboard dashboard = JacksonUtil.fromString(dashboardJson, Dashboard.class); Dashboard dashboard = JacksonUtil.fromString(dashboardJson, Dashboard.class);
assertThat(dashboard).isNotNull(); assertThat(dashboard).isNotNull();
assertThat(dashboard.getTitle()).containsIgnoringCase("gateway"); assertThat(dashboard.getTitle()).containsIgnoringCase("gateway");
assertThat(etag).isNotBlank(); assertThat(dashboard.getImage()).startsWith("tb-image;/api/images/system/gateway");
assertThat(response.getHeader("ETag")).isNotBlank();
}); });
} }

32
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@ -47,6 +47,7 @@ import java.util.Base64;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -145,8 +146,9 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
@After @After
public void afterTest() throws Exception { public void afterTest() throws Exception {
loginSysAdmin(); loginSysAdmin();
deleteTenant(savedTenant.getId()); deleteTenant(savedTenant.getId());
resourceService.deleteResourcesByTenantId(TenantId.SYS_TENANT_ID);
} }
@Test @Test
@ -277,30 +279,26 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
} }
@Test @Test
public void testSaveTbResourceWithExistsFileName() throws Exception { public void testSaveTbResourceWithSameFileName() throws Exception {
TbResource resource = new TbResource(); TbResource resource = new TbResource();
resource.setTenantId(tenantId); resource.setTenantId(tenantId);
resource.setResourceType(ResourceType.JKS); resource.setResourceType(ResourceType.JKS);
resource.setTitle("My resource"); resource.setTitle("My resource");
resource.setFileName(DEFAULT_FILE_NAME); resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA); resource.setData(TEST_DATA);
resource = tbResourceService.save(resource);
TbResource savedResource = tbResourceService.save(resource);
TbResource resource2 = new TbResource(); TbResource resource2 = new TbResource();
resource.setTenantId(tenantId); resource2.setTenantId(tenantId);
resource.setResourceType(ResourceType.JKS); resource2.setResourceType(ResourceType.JKS);
resource.setTitle("My resource"); resource2.setTitle("My resource");
resource.setFileName(DEFAULT_FILE_NAME); resource2.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA); resource2.setData(TEST_DATA);
resource2 = tbResourceService.save(resource2);
try {
Assertions.assertThrows(DataValidationException.class, () -> { assertThat(resource2.getId()).isNotEqualTo(resource.getId());
tbResourceService.save(resource2); assertThat(resource2.getFileName()).isEqualTo("test.jks");
}); assertThat(resource2.getResourceKey()).isEqualTo("test_(1).jks");
} finally {
tbResourceService.delete(savedResource, null);
}
} }
@Test @Test

2
common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java

@ -80,6 +80,4 @@ public interface ResourceService extends EntityDaoService {
TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data); TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data);
String checkSystemResourcesUsage(String content, ResourceType... usedResourceTypes);
} }

32
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@ -28,7 +28,6 @@ import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.RegexUtils;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey; import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent; import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent;
import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Dashboard;
@ -86,12 +85,15 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
); );
private static final Map<String, String> WIDGET_RESOURCES_MAPPING = Map.of( private static final Map<String, String> WIDGET_RESOURCES_MAPPING = Map.of(
"descriptor.resources.*.url.id", "" "resources.*.url.id", ""
); );
@Override @Override
public TbResource saveResource(TbResource resource, boolean doValidate) { public TbResource saveResource(TbResource resource, boolean doValidate) {
log.trace("Executing saveResource [{}]", resource); log.trace("Executing saveResource [{}]", resource);
if (resource.getTenantId() == null) {
resource.setTenantId(TenantId.SYS_TENANT_ID);
}
if (resource.getId() == null) { if (resource.getId() == null) {
resource.setResourceKey(getUniqueKey(resource.getTenantId(), resource.getResourceType(), StringUtils.defaultIfEmpty(resource.getResourceKey(), resource.getFileName()))); resource.setResourceKey(getUniqueKey(resource.getTenantId(), resource.getResourceType(), StringUtils.defaultIfEmpty(resource.getResourceKey(), resource.getFileName())));
} }
@ -317,6 +319,7 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
resourceInfo = findSystemOrTenantResourceByEtag(tenantId, resourceType, etag); resourceInfo = findSystemOrTenantResourceByEtag(tenantId, resourceType, etag);
if (resourceInfo == null) { if (resourceInfo == null) {
log.warn("[{}] Couldn't find resource referenced as '{}'", tenantId, value);
return value; return value;
} }
} else { // probably importing an old dashboard json where resources are referenced by ids } else { // probably importing an old dashboard json where resources are referenced by ids
@ -329,6 +332,7 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
resourceInfo = findResourceInfoById(tenantId, resourceId); resourceInfo = findResourceInfoById(tenantId, resourceId);
if (resourceInfo == null) { if (resourceInfo == null) {
updated.set(true); updated.set(true);
log.warn("[{}] Couldn't find resource referenced as '{}'", tenantId, value);
return ""; return "";
} }
} }
@ -386,18 +390,21 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
if (StringUtils.isBlank(value)) { if (StringUtils.isBlank(value)) {
return value; return value;
} }
return processor.apply(value); String newValue = processor.apply(value);
log.trace("Replaced '{}' with '{}'", value, newValue);
return newValue;
}); });
} }
@Override @Override
public TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data) { public TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data) {
if (resourceType == ResourceType.DASHBOARD) { if (resourceType == ResourceType.DASHBOARD) {
data = checkSystemResourcesUsage(data, ResourceType.JS_MODULE);
Dashboard dashboard = JacksonUtil.fromString(data, Dashboard.class); Dashboard dashboard = JacksonUtil.fromString(data, Dashboard.class);
dashboard.setTenantId(TenantId.SYS_TENANT_ID); dashboard.setTenantId(TenantId.SYS_TENANT_ID);
imageService.replaceBase64WithImageUrl(dashboard); imageService.replaceBase64WithImageUrl(dashboard);
replaceResourcesUsageWithUrls(dashboard);
data = JacksonUtil.toString(dashboard); data = JacksonUtil.toString(dashboard);
} }
@ -415,21 +422,6 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return saveResource(resource); return saveResource(resource);
} }
@Override
public String checkSystemResourcesUsage(String content, ResourceType... usedResourceTypes) {
return RegexUtils.replace(content, "\\$\\{RESOURCE:(.+)}", matchResult -> {
String resourceKey = matchResult.group(1);
for (ResourceType resourceType : usedResourceTypes) {
TbResourceInfo resource = findResourceInfoByTenantIdAndKey(TenantId.SYS_TENANT_ID, resourceType, resourceKey);
if (resource != null) {
log.trace("Replaced '{}' with resource id {}", matchResult.group(), resource.getUuidId());
return resource.getUuidId().toString();
}
}
return "";
});
}
@Override @Override
public String calculateEtag(byte[] data) { public String calculateEtag(byte[] data) {
return Hashing.sha256().hashBytes(data).toString(); return Hashing.sha256().hashBytes(data).toString();

11
dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java

@ -23,7 +23,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -212,7 +211,7 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService {
public void updateSystemWidgets(Stream<String> bundles, Stream<String> widgets) { public void updateSystemWidgets(Stream<String> bundles, Stream<String> widgets) {
widgets.forEach(widgetTypeJson -> { widgets.forEach(widgetTypeJson -> {
try { try {
updateSystemWidget(widgetTypeJson); updateSystemWidget(JacksonUtil.toJsonNode(widgetTypeJson));
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetTypeJson, e); throw new RuntimeException("Unable to load widget type from json: " + widgetTypeJson, e);
} }
@ -240,7 +239,7 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService {
JsonNode widgetTypesArrayJson = widgetsBundleDescriptor.get("widgetTypes"); JsonNode widgetTypesArrayJson = widgetsBundleDescriptor.get("widgetTypes");
widgetTypesArrayJson.forEach(widgetTypeJson -> { widgetTypesArrayJson.forEach(widgetTypeJson -> {
try { try {
WidgetTypeDetails widgetTypeDetails = updateSystemWidget(widgetTypeJson.toString()); WidgetTypeDetails widgetTypeDetails = updateSystemWidget(widgetTypeJson);
widgetTypeFqns.add(widgetTypeDetails.getFqn()); widgetTypeFqns.add(widgetTypeDetails.getFqn());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException("Unable to load widget type from json: " + widgetsBundleDescriptorJson, e); throw new RuntimeException("Unable to load widget type from json: " + widgetsBundleDescriptorJson, e);
@ -257,10 +256,8 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService {
}); });
} }
private WidgetTypeDetails updateSystemWidget(String widgetTypeJson) { private WidgetTypeDetails updateSystemWidget(JsonNode widgetTypeJson) {
widgetTypeJson = resourceService.checkSystemResourcesUsage(widgetTypeJson, ResourceType.JS_MODULE); WidgetTypeDetails widgetTypeDetails = JacksonUtil.treeToValue(widgetTypeJson, WidgetTypeDetails.class);
WidgetTypeDetails widgetTypeDetails = JacksonUtil.fromString(widgetTypeJson, WidgetTypeDetails.class);
WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(TenantId.SYS_TENANT_ID, widgetTypeDetails.getFqn()); WidgetType existingWidget = widgetTypeService.findWidgetTypeByTenantIdAndFqn(TenantId.SYS_TENANT_ID, widgetTypeDetails.getFqn());
if (existingWidget != null) { if (existingWidget != null) {
widgetTypeDetails.setId(existingWidget.getId()); widgetTypeDetails.setId(existingWidget.getId());

Loading…
Cancel
Save