Browse Source

Migrate resource's data from Base64 to bytea

pull/9524/head
ViacheslavKlimov 3 years ago
parent
commit
d53dbd6988
  1. 27
      application/src/main/data/upgrade/3.6.1/schema_update.sql
  2. 70
      application/src/main/java/org/thingsboard/server/controller/TbResourceController.java
  3. 1
      application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java
  4. 4
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  5. 5
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  6. 120
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  7. 15
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
  8. 10
      application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java
  9. 57
      application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java
  10. 58
      application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java
  11. 2
      application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java
  12. 51
      common/data/src/main/java/org/thingsboard/server/common/data/TbResource.java
  13. 37
      common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java
  14. 1
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  15. 3
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java
  16. 4
      dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java
  17. 6
      dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java
  18. 33
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
  19. 3
      dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java
  20. 22
      dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java
  21. 2
      dao/src/main/resources/sql/schema-entities.sql
  22. 3
      dao/src/test/java/org/thingsboard/server/dao/service/TenantServiceTest.java
  23. 4
      ui-ngx/src/app/core/http/resource.service.ts
  24. 16
      ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html
  25. 1
      ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts
  26. 2
      ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts
  27. 2
      ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html
  28. 12
      ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts
  29. 5
      ui-ngx/src/app/shared/models/resource.models.ts
  30. 2
      ui-ngx/src/app/shared/models/tenant.model.ts
  31. 9
      ui-ngx/src/assets/locale/locale.constant-en_US.json

27
application/src/main/data/upgrade/3.6.1/schema_update.sql

@ -0,0 +1,27 @@
--
-- Copyright © 2016-2023 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
DO
$$
BEGIN
IF NOT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_name = 'resource' AND column_name = 'data' AND data_type = 'bytea') THEN
ALTER TABLE resource RENAME COLUMN data TO base64_data;
ALTER TABLE resource ADD COLUMN data bytea;
UPDATE resource SET data = decode(base64_data, 'base64') WHERE base64_data IS NOT NULL;
ALTER TABLE resource DROP COLUMN base64_data;
END IF;
END;
$$;

70
application/src/main/java/org/thingsboard/server/controller/TbResourceController.java

@ -26,7 +26,10 @@ import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
@ -87,15 +90,15 @@ public class TbResourceController extends BaseController {
@ApiOperation(value = "Download Resource (downloadResource)", notes = "Download Resource based on the provided Resource Id." + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/{resourceId}/download", method = RequestMethod.GET)
@GetMapping(value = "/resource/{resourceId}/download")
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadResource(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId) throws ThingsboardException {
public ResponseEntity<ByteArrayResource> downloadResource(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId) throws ThingsboardException {
checkParameter(RESOURCE_ID, strResourceId);
TbResourceId resourceId = new TbResourceId(toUUID(strResourceId));
TbResource tbResource = checkResourceId(resourceId, Operation.READ);
ByteArrayResource resource = new ByteArrayResource(Base64.getDecoder().decode(tbResource.getData().getBytes()));
ByteArrayResource resource = new ByteArrayResource(tbResource.getData());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + tbResource.getFileName())
.header("x-filename", tbResource.getFileName())
@ -106,11 +109,11 @@ public class TbResourceController extends BaseController {
@ApiOperation(value = "Download LWM2M Resource (downloadLwm2mResourceIfChanged)", notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/lwm2m/{resourceId}/download", method = RequestMethod.GET, produces = "application/xml")
@GetMapping(value = "/resource/lwm2m/{resourceId}/download", produces = "application/xml")
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadLwm2mResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
public ResponseEntity<ByteArrayResource> downloadLwm2mResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
return downloadResourceIfChanged(ResourceType.LWM2M_MODEL, strResourceId, etag);
}
@ -118,30 +121,30 @@ public class TbResourceController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/pkcs12/{resourceId}/download", method = RequestMethod.GET, produces = "application/x-pkcs12")
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadPkcs12ResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
public ResponseEntity<ByteArrayResource> downloadPkcs12ResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
return downloadResourceIfChanged(ResourceType.PKCS_12, strResourceId, etag);
}
@ApiOperation(value = "Download JKS Resource (downloadJksResourceIfChanged)",
notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/jks/{resourceId}/download", method = RequestMethod.GET, produces = "application/x-java-keystore")
@GetMapping(value = "/resource/jks/{resourceId}/download", produces = "application/x-java-keystore")
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadJksResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
public ResponseEntity<ByteArrayResource> downloadJksResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
return downloadResourceIfChanged(ResourceType.JKS, strResourceId, etag);
}
@ApiOperation(value = "Download JS Resource (downloadJsResourceIfChanged)", notes = DOWNLOAD_RESOURCE_IF_NOT_CHANGED + AVAILABLE_FOR_ANY_AUTHORIZED_USER)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/resource/js/{resourceId}/download", method = RequestMethod.GET, produces = "application/javascript")
@GetMapping(value = "/resource/js/{resourceId}/download", produces = "application/javascript")
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadJsResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
public ResponseEntity<ByteArrayResource> downloadJsResourceIfChanged(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws ThingsboardException {
return downloadResourceIfChanged(ResourceType.JS_MODULE, strResourceId, etag);
}
@ -150,7 +153,7 @@ public class TbResourceController extends BaseController {
RESOURCE_INFO_DESCRIPTION + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH,
produces = "application/json")
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/info/{resourceId}", method = RequestMethod.GET)
@GetMapping(value = "/resource/info/{resourceId}")
@ResponseBody
public TbResourceInfo getResourceInfoById(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId) throws ThingsboardException {
@ -162,15 +165,18 @@ public class TbResourceController extends BaseController {
@ApiOperation(value = "Get Resource (getResourceById)",
notes = "Fetch the Resource object based on the provided Resource Id. " +
RESOURCE_DESCRIPTION + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH,
produces = "application/json")
produces = "application/json", hidden = true)
@Deprecated // resource's data should be fetched with a download request
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/{resourceId}", method = RequestMethod.GET)
@GetMapping(value = "/resource/{resourceId}")
@ResponseBody
public TbResource getResourceById(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable(RESOURCE_ID) String strResourceId) throws ThingsboardException {
checkParameter(RESOURCE_ID, strResourceId);
TbResourceId resourceId = new TbResourceId(toUUID(strResourceId));
return checkResourceId(resourceId, Operation.READ);
TbResource resource = checkResourceId(resourceId, Operation.READ);
resource.setBase64Data(Base64.getEncoder().encodeToString(resource.getData()));
return resource;
}
@ApiOperation(value = "Create Or Update Resource (saveResource)",
@ -184,10 +190,10 @@ public class TbResourceController extends BaseController {
produces = "application/json",
consumes = "application/json")
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource", method = RequestMethod.POST)
@PostMapping(value = "/resource")
@ResponseBody
public TbResource saveResource(@ApiParam(value = "A JSON value representing the Resource.")
@RequestBody TbResource resource) throws Exception {
public TbResourceInfo saveResource(@ApiParam(value = "A JSON value representing the Resource.")
@RequestBody TbResource resource) throws Exception {
resource.setTenantId(getTenantId());
checkEntity(resource.getId(), resource, Resource.TB_RESOURCE);
return tbResourceService.save(resource, getCurrentUser());
@ -198,7 +204,7 @@ public class TbResourceController extends BaseController {
PAGE_DATA_PARAMETERS + RESOURCE_INFO_DESCRIPTION + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH,
produces = "application/json")
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource", method = RequestMethod.GET)
@GetMapping(value = "/resource")
@ResponseBody
public PageData<TbResourceInfo> getResources(@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ -230,7 +236,7 @@ public class TbResourceController extends BaseController {
PAGE_DATA_PARAMETERS + LWM2M_OBJECT_DESCRIPTION + TENANT_AUTHORITY_PARAGRAPH,
produces = "application/json")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/resource/lwm2m/page", method = RequestMethod.GET)
@GetMapping(value = "/resource/lwm2m/page")
@ResponseBody
public List<LwM2mObject> getLwm2mListObjectsPage(@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ -251,7 +257,7 @@ public class TbResourceController extends BaseController {
"You can specify parameters to filter the results. " + LWM2M_OBJECT_DESCRIPTION + TENANT_AUTHORITY_PARAGRAPH,
produces = "application/json")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/resource/lwm2m", method = RequestMethod.GET)
@GetMapping(value = "/resource/lwm2m")
@ResponseBody
public List<LwM2mObject> getLwm2mListObjects(@ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES, required = true)
@RequestParam String sortOrder,
@ -265,7 +271,7 @@ public class TbResourceController extends BaseController {
@ApiOperation(value = "Delete Resource (deleteResource)",
notes = "Deletes the Resource. Referencing non-existing Resource Id will cause an error." + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/resource/{resourceId}", method = RequestMethod.DELETE)
@DeleteMapping(value = "/resource/{resourceId}")
@ResponseBody
public void deleteResource(@ApiParam(value = RESOURCE_ID_PARAM_DESCRIPTION)
@PathVariable("resourceId") String strResourceId) throws ThingsboardException {
@ -275,7 +281,7 @@ public class TbResourceController extends BaseController {
tbResourceService.delete(tbResource, getCurrentUser());
}
private ResponseEntity<org.springframework.core.io.Resource> downloadResourceIfChanged(ResourceType type, String strResourceId, String etag) throws ThingsboardException {
private ResponseEntity<ByteArrayResource> downloadResourceIfChanged(ResourceType type, String strResourceId, String etag) throws ThingsboardException {
checkParameter(RESOURCE_ID, strResourceId);
TbResourceId resourceId = new TbResourceId(toUUID(strResourceId));
@ -289,7 +295,7 @@ public class TbResourceController extends BaseController {
}
TbResource tbResource = checkResourceId(resourceId, Operation.READ);
ByteArrayResource resource = new ByteArrayResource(Base64.getDecoder().decode(tbResource.getData().getBytes()));
ByteArrayResource resource = new ByteArrayResource(tbResource.getData());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + tbResource.getFileName())

1
application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java

@ -133,6 +133,7 @@ public class TenantProfileController extends BaseController {
" \"maxRuleChains\": 0,\n" +
" \"maxResourcesInBytes\": 0,\n" +
" \"maxOtaPackagesInBytes\": 0,\n" +
" \"maxResourceSize\": 0,\n" +
" \"transportTenantMsgRateLimit\": \"1000:1,20000:60\",\n" +
" \"transportTenantTelemetryMsgRateLimit\": \"1000:1,20000:60\",\n" +
" \"transportTenantTelemetryDataPointsRateLimit\": \"1000:1,20000:60\",\n" +

4
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -265,6 +265,10 @@ public class ThingsboardInstallService {
case "3.6.0":
log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.0");
break;
case "3.6.1":
log.info("Upgrading ThingsBoard from version 3.6.1 to 3.6.2 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.1");
//TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
break;
default:

5
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@ -47,7 +47,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
@ -309,7 +308,7 @@ public class InstallScripts {
dirStream.forEach(
path -> {
try {
String data = Base64.getEncoder().encodeToString(Files.readAllBytes(path));
byte[] data = Files.readAllBytes(path);
TbResource tbResource = new TbResource();
tbResource.setTenantId(TenantId.SYS_TENANT_ID);
tbResource.setData(data);
@ -330,7 +329,7 @@ public class InstallScripts {
private void doSaveLwm2mResource(TbResource resource) throws ThingsboardException {
log.trace("Executing saveResource [{}]", resource);
if (StringUtils.isEmpty(resource.getData())) {
if (resource.getData() == null || resource.getData().length == 0) {
throw new DataValidationException("Resource data should be specified!");
}
toLwm2mResource(resource);

120
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -63,6 +63,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO;
import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS;
@ -717,86 +718,73 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
break;
case "3.5.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
if (isOldSchema(conn, 3005000)) {
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.5.0", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3005001;");
}
log.info("Schema updated.");
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
updateSchema("3.5.0", 3005000, "3.5.1", 3005001, null);
break;
case "3.5.1":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
if (isOldSchema(conn, 3005001)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.5.1", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
String[] entityNames = new String[]{"device", "component_descriptor", "customer", "dashboard", "rule_chain", "rule_node", "ota_package",
"asset_profile", "asset", "device_profile", "tb_user", "tenant_profile", "tenant", "widgets_bundle", "entity_view", "edge"};
for (String entityName : entityNames) {
try {
conn.createStatement().execute("ALTER TABLE " + entityName + " DROP COLUMN " + SEARCH_TEXT + " CASCADE");
} catch (Exception e) {
}
}
updateSchema("3.5.1", 3005001, "3.6.0", 3006000, conn -> {
String[] entityNames = new String[]{"device", "component_descriptor", "customer", "dashboard", "rule_chain", "rule_node", "ota_package",
"asset_profile", "asset", "device_profile", "tb_user", "tenant_profile", "tenant", "widgets_bundle", "entity_view", "edge"};
for (String entityName : entityNames) {
try {
conn.createStatement().execute("ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS configuration_version int DEFAULT 0;");
conn.createStatement().execute("ALTER TABLE " + entityName + " DROP COLUMN " + SEARCH_TEXT + " CASCADE");
} catch (Exception e) {
}
try {
conn.createStatement().execute("ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS configuration_version int DEFAULT 0;");
} catch (Exception e) {
}
try {
conn.createStatement().execute("CREATE INDEX IF NOT EXISTS idx_rule_node_type_configuration_version ON rule_node(type, configuration_version);");
} catch (Exception e) {
}
try {
conn.createStatement().execute("UPDATE rule_node SET " +
"configuration = (configuration::jsonb || '{\"updateAttributesOnlyOnValueChange\": \"false\"}'::jsonb)::varchar, " +
"configuration_version = 1 " +
"WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode' AND configuration_version < 1;");
} catch (Exception e) {
}
try {
conn.createStatement().execute("CREATE INDEX IF NOT EXISTS idx_notification_recipient_id_unread ON notification(recipient_id) WHERE status <> 'READ';");
} catch (Exception e) {
}
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3006000;");
log.info("Schema updated to version 3.6.0.");
} else {
log.info("Skip schema re-update to version 3.6.0. Use env flag 'SKIP_SCHEMA_VERSION_CHECK' to force the re-update.");
}
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
try {
conn.createStatement().execute("ALTER TABLE component_descriptor ADD COLUMN IF NOT EXISTS configuration_version int DEFAULT 0;");
} catch (Exception e) {
}
try {
conn.createStatement().execute("ALTER TABLE rule_node ADD COLUMN IF NOT EXISTS configuration_version int DEFAULT 0;");
} catch (Exception e) {
}
try {
conn.createStatement().execute("CREATE INDEX IF NOT EXISTS idx_rule_node_type_configuration_version ON rule_node(type, configuration_version);");
} catch (Exception e) {
}
try {
conn.createStatement().execute("UPDATE rule_node SET " +
"configuration = (configuration::jsonb || '{\"updateAttributesOnlyOnValueChange\": \"false\"}'::jsonb)::varchar, " +
"configuration_version = 1 " +
"WHERE type = 'org.thingsboard.rule.engine.telemetry.TbMsgAttributesNode' AND configuration_version < 1;");
} catch (Exception e) {
}
try {
conn.createStatement().execute("CREATE INDEX IF NOT EXISTS idx_notification_recipient_id_unread ON notification(recipient_id) WHERE status <> 'READ';");
} catch (Exception e) {
}
});
break;
case "3.6.0":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
if (isOldSchema(conn, 3006000)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.6.0", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3006001;");
log.info("Schema updated to version 3.6.1.");
} else {
log.info("Skip schema re-update to version 3.6.1. Use env flag 'SKIP_SCHEMA_VERSION_CHECK' to force the re-update.");
}
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
updateSchema("3.6.0", 3006000, "3.6.1", 3006001, null);
break;
case "3.6.1":
updateSchema("3.6.1", 3006001, "3.6.2", 3006002, null);
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}
}
private void updateSchema(String oldVersionStr, int oldVersion, String newVersionStr, int newVersion, Consumer<Connection> additionalAction) {
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
if (isOldSchema(conn, oldVersion)) {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", oldVersionStr, SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
if (additionalAction != null) {
additionalAction.accept(conn);
}
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = " + newVersion + ";");
log.info("Schema updated to version {}", newVersionStr);
} else {
log.info("Skip schema re-update to version {}. Use env flag 'SKIP_SCHEMA_VERSION_CHECK' to force the re-update.", newVersionStr);
}
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
}
private void runSchemaUpdateScript(Connection connection, String version) throws Exception {
Path schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", version, SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, connection);

15
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@ -15,34 +15,26 @@
*/
package org.thingsboard.server.service.resource;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hashing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.TbResourceInfoFilter;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.lwm2m.LwM2mObject;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.widget.BaseWidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import java.util.Base64;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;
@ -166,18 +158,11 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
}
private TbResource doSave(TbResource resource) throws ThingsboardException {
log.trace("Executing saveResource [{}]", resource);
if (StringUtils.isEmpty(resource.getData())) {
throw new DataValidationException("Resource data should be specified!");
}
if (ResourceType.LWM2M_MODEL.equals(resource.getResourceType())) {
toLwm2mResource(resource);
} else {
resource.setResourceKey(resource.getFileName());
}
HashCode hashCode = Hashing.sha256().hashBytes(Base64.getDecoder().decode(resource.getData().getBytes()));
resource.setEtag(hashCode.toString());
return resourceService.saveResource(resource);
}
}

10
application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java

@ -39,13 +39,13 @@ import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPA
@Slf4j
public class LwM2mObjectModelUtils {
private static final TbDDFFileParser ddfFileParser = new TbDDFFileParser();
public static void toLwm2mResource (TbResource resource) throws ThingsboardException {
public static void toLwm2mResource(TbResource resource) throws ThingsboardException {
try {
List<ObjectModel> objectModels =
ddfFileParser.parse(new ByteArrayInputStream(Base64.getDecoder().decode(resource.getData())), resource.getSearchText());
ddfFileParser.parse(new ByteArrayInputStream(resource.getData()), resource.getSearchText());
if (!objectModels.isEmpty()) {
ObjectModel objectModel = objectModels.get(0);
@ -73,7 +73,7 @@ public class LwM2mObjectModelUtils {
public static LwM2mObject toLwM2mObject(TbResource resource, boolean isSave) {
try {
List<ObjectModel> objectModels =
ddfFileParser.parse(new ByteArrayInputStream(Base64.getDecoder().decode(resource.getData())), resource.getSearchText());
ddfFileParser.parse(new ByteArrayInputStream(resource.getData()), resource.getSearchText());
if (objectModels.size() == 0) {
return null;
} else {

57
application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.http.HttpHeaders;
@ -35,11 +34,11 @@ import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DaoSqlTest;
@ -54,6 +53,11 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@DaoSqlTest
public class TbResourceControllerTest extends AbstractControllerTest {
/*
* TODO:
* test maxResourceSize limit in tenant profile
* */
private IdComparator<TbResourceInfo> idComparator = new IdComparator<>();
private static final String DEFAULT_FILE_NAME = "test.jks";
@ -101,7 +105,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle("My first resource");
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -116,7 +120,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
Assert.assertEquals(resource.getTitle(), savedResource.getTitle());
Assert.assertEquals(DEFAULT_FILE_NAME, savedResource.getFileName());
Assert.assertEquals(DEFAULT_FILE_NAME, savedResource.getResourceKey());
Assert.assertEquals(resource.getData(), savedResource.getData());
Assert.assertArrayEquals(resource.getData(), download(savedResource.getId()));
savedResource.setTitle("My new resource");
@ -136,7 +140,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle(StringUtils.randomAlphabetic(300));
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
Mockito.reset(tbClusterService, auditLogService);
@ -155,7 +159,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle("My first resource");
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -184,13 +188,14 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle("My first resource");
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
TbResource foundResource = doGet("/api/resource/" + savedResource.getId().getId().toString(), TbResource.class);
TbResource foundResource = doGet("/api/resource/" + savedResource.getUuidId(), TbResource.class);
Assert.assertNotNull(foundResource);
Assert.assertEquals(savedResource, foundResource);
Assert.assertEquals(savedResource.getId(), foundResource.getId());
Assert.assertEquals(savedResource.getFileName(), foundResource.getFileName());
}
@Test
@ -199,7 +204,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle("My first resource");
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -213,7 +218,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
ActionType.DELETED, resourceIdStr);
doGet("/api/resource/" + savedResource.getId().getId().toString())
doGet("/api/resource/" + savedResource.getUuidId())
.andExpect(status().isNotFound())
.andExpect(statusReason(containsString(msgErrorNoFound("Resource", resourceIdStr))));
}
@ -224,7 +229,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JKS);
resource.setTitle("My first resource");
resource.setFileName(DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -255,7 +260,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
resources.add(new TbResourceInfo(save(resource)));
}
List<TbResourceInfo> loadedResources = new ArrayList<>();
@ -292,7 +297,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("JKS Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
resources.add(new TbResourceInfo(save(resource)));
}
@ -302,7 +307,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("LWM2M Resource" + i);
resource.setResourceType(ResourceType.PKCS_12);
resource.setFileName(i + DEFAULT_FILE_NAME_2);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
save(resource);
}
@ -339,7 +344,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
resources.add(new TbResourceInfo(save(resource)));
}
List<TbResourceInfo> loadedResources = new ArrayList<>();
@ -399,7 +404,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("JKS Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResourceInfo saved = new TbResourceInfo(save(resource));
jksResources.add(saved);
}
@ -410,7 +415,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("LWM2M Resource" + i);
resource.setResourceType(ResourceType.PKCS_12);
resource.setFileName(i + DEFAULT_FILE_NAME_2);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource saved = save(resource);
lwm2mesources.add(saved);
}
@ -476,7 +481,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
expectedResources.add(new TbResourceInfo(save(resource)));
}
@ -487,7 +492,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setTitle("Resource" + i);
resource.setResourceType(ResourceType.JKS);
resource.setFileName(i + DEFAULT_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResourceInfo savedResource = new TbResourceInfo(save(resource));
systemResources.add(savedResource);
if (i >= 73) {
@ -531,7 +536,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JS_MODULE);
resource.setTitle("Js resource");
resource.setFileName(JS_TEST_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -562,7 +567,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JS_MODULE);
resource.setTitle("Js resource");
resource.setFileName(JS_TEST_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -606,7 +611,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
resource.setResourceType(ResourceType.JS_MODULE);
resource.setTitle("Js resource");
resource.setFileName(JS_TEST_FILE_NAME);
resource.setData(TEST_DATA);
resource.setBase64Data(TEST_DATA);
TbResource savedResource = save(resource);
@ -619,4 +624,10 @@ public class TbResourceControllerTest extends AbstractControllerTest {
private TbResource save(TbResource tbResource) throws Exception {
return doPostWithTypedResponse("/api/resource", tbResource, new TypeReference<>(){});
}
private byte[] download(TbResourceId resourceId) throws Exception {
return doGet("/api/resource/" + resourceId + "/download")
.andExpect(status().isOk())
.andReturn().getResponse().getContentAsByteArray();
}
}

58
application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

File diff suppressed because one or more lines are too long

2
application/src/test/java/org/thingsboard/server/transport/lwm2m/AbstractLwM2MIntegrationTest.java

@ -206,7 +206,7 @@ public abstract class AbstractLwM2MIntegrationTest extends AbstractTransportInte
lwModel.setFileName(resourceName);
lwModel.setTenantId(tenantId);
byte[] bytes = IOUtils.toByteArray(AbstractLwM2MIntegrationTest.class.getClassLoader().getResourceAsStream("lwm2m/" + resourceName));
lwModel.setData(Base64.getEncoder().encodeToString(bytes));
lwModel.setBase64Data(Base64.getEncoder().encodeToString(bytes));
lwModel = doPostWithTypedResponse("/api/resource", lwModel, new TypeReference<>() {
});
Assert.assertNotNull(lwModel);

51
common/data/src/main/java/org/thingsboard/server/common/data/TbResource.java

File diff suppressed because one or more lines are too long

37
common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java

@ -50,6 +50,10 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
private String searchText;
@ApiModelProperty(position = 8, value = "Resource etag.", example = "33a64df551425fcc55e4d42a148795d9f25f89d4", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String etag;
@NoXss
@Length(fieldName = "file name")
@ApiModelProperty(position = 9, value = "Resource file name.", example = "19.xml", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String fileName;
public TbResourceInfo() {
super();
@ -61,12 +65,13 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
public TbResourceInfo(TbResourceInfo resourceInfo) {
super(resourceInfo);
this.tenantId = resourceInfo.getTenantId();
this.title = resourceInfo.getTitle();
this.resourceType = resourceInfo.getResourceType();
this.resourceKey = resourceInfo.getResourceKey();
this.searchText = resourceInfo.getSearchText();
this.etag = resourceInfo.getEtag();
this.tenantId = resourceInfo.tenantId;
this.title = resourceInfo.title;
this.resourceType = resourceInfo.resourceType;
this.resourceKey = resourceInfo.resourceKey;
this.searchText = resourceInfo.searchText;
this.etag = resourceInfo.etag;
this.fileName = resourceInfo.fileName;
}
@ApiModelProperty(position = 1, value = "JSON object with the Resource Id. " +
@ -95,24 +100,4 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
return searchText != null ? searchText : title;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("ResourceInfo [tenantId=");
builder.append(tenantId);
builder.append(", id=");
builder.append(getUuidId());
builder.append(", createdTime=");
builder.append(createdTime);
builder.append(", title=");
builder.append(title);
builder.append(", resourceType=");
builder.append(resourceType);
builder.append(", resourceKey=");
builder.append(resourceKey);
builder.append(", hashCode=");
builder.append(etag);
builder.append("]");
return builder.toString();
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java

@ -39,6 +39,7 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura
private long maxRuleChains;
private long maxResourcesInBytes;
private long maxOtaPackagesInBytes;
private long maxResourceSize;
private String transportTenantMsgRateLimit;
private String transportTenantTelemetryMsgRateLimit;

3
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java

@ -152,8 +152,7 @@ public class LwM2mVersionedModelProvider implements LwM2mModelProvider {
private ObjectModel getObjectModel(String key) {
Optional<TbResource> tbResource = context.getTransportResourceCache().get(this.tenantId, LWM2M_MODEL, key);
return tbResource.map(resource -> helper.parseFromXmlToObjectModel(
Base64.getDecoder().decode(resource.getData()),
return tbResource.map(resource -> helper.parseFromXmlToObjectModel(resource.getData(),
key + ".xml")).orElse(null);
}
}

4
dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java

@ -30,8 +30,8 @@ import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DATA_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN;
@ -64,7 +64,7 @@ public class TbResourceEntity extends BaseSqlEntity<TbResource> implements BaseE
private String fileName;
@Column(name = RESOURCE_DATA_COLUMN)
private String data;
private byte[] data;
@Column(name = RESOURCE_ETAG_COLUMN)
private String etag;

6
dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java

@ -30,6 +30,7 @@ import javax.persistence.Table;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN;
@ -61,6 +62,9 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
@Column(name = RESOURCE_ETAG_COLUMN)
private String hashCode;
@Column(name = RESOURCE_FILE_NAME_COLUMN)
private String fileName;
public TbResourceInfoEntity() {
}
@ -75,6 +79,7 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
this.resourceKey = resource.getResourceKey();
this.searchText = resource.getSearchText();
this.hashCode = resource.getEtag();
this.fileName = resource.getFileName();
}
@Override
@ -87,6 +92,7 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
resource.setResourceKey(resourceKey);
resource.setSearchText(searchText);
resource.setEtag(hashCode);
resource.setFileName(fileName);
return resource;
}
}

33
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@ -15,16 +15,17 @@
*/
package org.thingsboard.server.dao.resource;
import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.server.cache.device.DeviceCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoEvictEvent;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.cache.resourceInfo.ResourceInfoCacheKey;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TbResourceInfo;
@ -43,6 +44,7 @@ import org.thingsboard.server.dao.service.Validator;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import static org.thingsboard.server.dao.device.DeviceServiceImpl.INCORRECT_TENANT_ID;
import static org.thingsboard.server.dao.service.Validator.validateId;
@ -59,9 +61,26 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
@Override
public TbResource saveResource(TbResource resource) {
log.trace("Executing saveResource [{}]", resource);
if (resource.getData() != null) {
resource.setEtag(calculateEtag(resource.getData()));
}
resourceValidator.validate(resource, TbResourceInfo::getTenantId);
boolean newResource = resource.getId() == null;
if (newResource) {
UUID uuid = Uuids.timeBased();
resource.setId(new TbResourceId(uuid));
resource.setCreatedTime(Uuids.unixTimestamp(uuid));
}
try {
TbResource saved = resourceDao.save(resource.getTenantId(), resource);
TbResource saved;
if (resource.getData() != null) {
saved = resourceDao.save(resource.getTenantId(), resource);
} else {
TbResourceInfo resourceInfo = saveResourceInfo(resource);
saved = new TbResource(resourceInfo);
}
publishEvictEvent(new ResourceInfoEvictEvent(resource.getTenantId(), resource.getId()));
return saved;
} catch (Exception t) {
@ -76,6 +95,10 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
}
private TbResourceInfo saveResourceInfo(TbResource resource) {
return resourceInfoDao.save(resource.getTenantId(), new TbResourceInfo(resource));
}
@Override
public TbResource getResource(TenantId tenantId, ResourceType resourceType, String resourceKey) {
log.trace("Executing getResource [{}] [{}] [{}]", tenantId, resourceType, resourceKey);
@ -165,6 +188,10 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return resourceDao.sumDataSizeByTenantId(tenantId);
}
private String calculateEtag(byte[] data) {
return Hashing.sha256().hashBytes(data).toString();
}
private final PaginatedRemover<TenantId, TbResource> tenantResourcesRemover =
new PaginatedRemover<>() {

3
dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java

@ -128,8 +128,7 @@ public abstract class DataValidator<D extends BaseData<?>> {
EntityType entityType) {
if (maxSumDataSize > 0) {
if (dataDao.sumDataSizeByTenantId(tenantId) + currentDataSize > maxSumDataSize) {
throw new DataValidationException(String.format("Failed to create the %s, files size limit is exhausted %d bytes!",
entityType.name().toLowerCase().replaceAll("_", " "), maxSumDataSize));
throw new DataValidationException(String.format("%ss total size exceeds the maximum of " + maxSumDataSize + " bytes", entityType.getNormalName()));
}
}
}

22
dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java

@ -56,12 +56,28 @@ public class ResourceDataValidator extends DataValidator<TbResource> {
@Override
protected void validateCreate(TenantId tenantId, TbResource resource) {
if (resource.getData() == null || resource.getData().length == 0) {
throw new DataValidationException("Resource data should be specified");
}
if (tenantId != null && !TenantId.SYS_TENANT_ID.equals(tenantId)) {
DefaultTenantProfileConfiguration profileConfiguration =
(DefaultTenantProfileConfiguration) tenantProfileCache.get(tenantId).getProfileData().getConfiguration();
DefaultTenantProfileConfiguration profileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration();
long maxResourceSize = profileConfiguration.getMaxResourceSize();
if (maxResourceSize > 0 && resource.getData().length > maxResourceSize) {
throw new IllegalArgumentException("Resource exceeds the maximum size of " + maxResourceSize + " bytes");
}
long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes();
validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, resource.getData().length(), TB_RESOURCE);
validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, resource.getData().length, TB_RESOURCE);
}
}
@Override
protected TbResource validateUpdate(TenantId tenantId, TbResource resource) {
if (!tenantId.isSysTenantId()) {
if (resource.getData() != null) {
throw new DataValidationException("Resource data can't be updated");
}
}
return resource;
}
@Override

2
dao/src/main/resources/sql/schema-entities.sql

@ -713,7 +713,7 @@ CREATE TABLE IF NOT EXISTS resource (
resource_key varchar(255) NOT NULL,
search_text varchar(255),
file_name varchar(255) NOT NULL,
data varchar,
data bytea,
etag varchar,
CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key)
);

3
dao/src/test/java/org/thingsboard/server/dao/service/TenantServiceTest.java

@ -73,6 +73,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -615,7 +616,7 @@ public class TenantServiceTest extends AbstractServiceTest {
resource.setResourceType(ResourceType.LWM2M_MODEL);
resource.setFileName("filename.txt");
resource.setResourceKey("Test resource key");
resource.setData("Some super test data");
resource.setData("Some super test data".getBytes(StandardCharsets.UTF_8));
return resourceService.saveResource(resource);
}

4
ui-ngx/src/app/core/http/resource.service.ts

@ -42,10 +42,6 @@ export class ResourceService {
return this.http.get<PageData<ResourceInfo>>(url, defaultHttpOptionsFromConfig(config));
}
public getResource(resourceId: string, config?: RequestConfig): Observable<Resource> {
return this.http.get<Resource>(`/api/resource/${resourceId}`, defaultHttpOptionsFromConfig(config));
}
public getResourceInfo(resourceId: string, config?: RequestConfig): Observable<ResourceInfo> {
return this.http.get<Resource>(`/api/resource/info/${resourceId}`, defaultHttpOptionsFromConfig(config));
}

16
ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html

@ -386,6 +386,22 @@
<mat-hint></mat-hint>
</mat-form-field>
</div>
<div fxFlex fxLayout="row" fxLayout.xs="column" fxLayoutGap.gt-xs="16px">
<mat-form-field fxFlex class="mat-block" appearance="fill">
<mat-label translate>tenant-profile.maximum-resource-size</mat-label>
<input matInput required min="0" step="1"
formControlName="maxResourceSize"
type="number">
<mat-error *ngIf="defaultTenantProfileConfigurationFormGroup.get('maxResourceSize').hasError('required')">
{{ 'tenant-profile.maximum-resource-size-required' | translate}}
</mat-error>
<mat-error *ngIf="defaultTenantProfileConfigurationFormGroup.get('maxResourceSize').hasError('min')">
{{ 'tenant-profile.maximum-resource-size-range' | translate}}
</mat-error>
<mat-hint></mat-hint>
</mat-form-field>
<div fxFlex></div>
</div>
</fieldset>
<fieldset class="fields-group">

1
ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts

@ -67,6 +67,7 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA
maxRuleChains: [null, [Validators.required, Validators.min(0)]],
maxResourcesInBytes: [null, [Validators.required, Validators.min(0)]],
maxOtaPackagesInBytes: [null, [Validators.required, Validators.min(0)]],
maxResourceSize: [null, [Validators.required, Validators.min(0)]],
transportTenantMsgRateLimit: [null, []],
transportTenantTelemetryMsgRateLimit: [null, []],
transportTenantTelemetryDataPointsRateLimit: [null, []],

2
ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts

@ -84,7 +84,7 @@ export class ResourcesLibraryTableConfigResolver implements Resolve<EntityTableC
this.config.deleteEntitiesContent = () => this.translate.instant('resource.delete-resources-text');
this.config.entitiesFetchFunction = pageLink => this.resourceService.getResources(pageLink, this.config.componentsData.resourceType);
this.config.loadEntity = id => this.resourceService.getResource(id.id);
this.config.loadEntity = id => this.resourceService.getResourceInfo(id.id);
this.config.saveEntity = resource => this.saveResource(resource);
this.config.deleteEntity = id => this.resourceService.deleteResource(id.id);

2
ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html

@ -67,7 +67,7 @@
</mat-error>
</mat-form-field>
<tb-file-input *ngIf="isAdd || (isEdit && entityForm.get('resourceType').value === resourceType.JS_MODULE)"
formControlName="data"
formControlName="base64Data"
required
[readAsBinary]="true"
[allowedExtensions]="getAllowedExtensions()"

12
ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts

@ -68,7 +68,7 @@ export class ResourcesLibraryComponent extends EntityComponent<Resource> impleme
this.entityForm.get('title').enable({emitEvent: false});
}
this.entityForm.patchValue({
data: null,
base64Data: null,
fileName: null
}, {emitEvent: false});
});
@ -93,7 +93,7 @@ export class ResourcesLibraryComponent extends EntityComponent<Resource> impleme
title: [entity ? entity.title : '', [Validators.required, Validators.maxLength(255)]],
resourceType: [entity?.resourceType ? entity.resourceType : ResourceType.JS_MODULE, Validators.required],
fileName: [entity ? entity.fileName : null, Validators.required],
data: [entity ? entity.data : null, Validators.required]
base64Data: [entity ? entity.base64Data : null, Validators.required]
});
}
@ -102,20 +102,20 @@ export class ResourcesLibraryComponent extends EntityComponent<Resource> impleme
this.entityForm.get('resourceType').disable({emitEvent: false});
if (entity.resourceType !== ResourceType.JS_MODULE) {
this.entityForm.get('fileName').disable({emitEvent: false});
this.entityForm.get('data').disable({emitEvent: false});
this.entityForm.get('base64Data').disable({emitEvent: false});
}
}
this.entityForm.patchValue({
resourceType: entity.resourceType,
fileName: entity.fileName,
title: entity.title,
data: entity.data
base64Data: entity.base64Data
});
}
prepareFormValue(formValue: Resource): Resource {
if (this.isEdit && !isDefinedAndNotNull(formValue.data)) {
delete formValue.data;
if (this.isEdit && !isDefinedAndNotNull(formValue.base64Data)) {
delete formValue.base64Data;
}
return super.prepareFormValue(formValue);
}

5
ui-ngx/src/app/shared/models/resource.models.ts

@ -57,14 +57,15 @@ export interface ResourceInfo extends Omit<BaseData<TbResourceId>, 'name' | 'lab
resourceKey?: string;
title?: string;
resourceType: ResourceType;
fileName: string;
}
export interface Resource extends ResourceInfo {
data: string;
fileName: string;
base64Data: string;
name?: string;
}
// @ts-ignore FIXME
export interface Resources extends ResourceInfo {
data: Array<string>;
fileName: Array<string>;

2
ui-ngx/src/app/shared/models/tenant.model.ts

@ -33,6 +33,7 @@ export interface DefaultTenantProfileConfiguration {
maxRuleChains: number;
maxResourcesInBytes: number;
maxOtaPackagesInBytes: number;
maxResourceSize: number;
transportTenantMsgRateLimit?: string;
transportTenantTelemetryMsgRateLimit?: string;
@ -101,6 +102,7 @@ export function createTenantProfileConfiguration(type: TenantProfileType): Tenan
maxRuleChains: 0,
maxResourcesInBytes: 0,
maxOtaPackagesInBytes: 0,
maxResourceSize: 0,
maxTransportMessages: 0,
maxTransportDataPoints: 0,
maxREExecutions: 0,

9
ui-ngx/src/assets/locale/locale.constant-en_US.json

@ -3948,7 +3948,7 @@
"rule-engine": "Rule Engine",
"time-to-live": "Time-to-live",
"alarms-and-notifications": "Alarms and notifications",
"ota-files-in-bytes": "OTA files in bytes",
"ota-files-in-bytes": "Files",
"ws-title": "WS",
"unlimited": "(0 - unlimited)",
"maximum-devices": "Devices maximum number",
@ -3971,10 +3971,13 @@
"maximum-rule-chains-range": "Rule chains maximum number can't be negative",
"maximum-resources-sum-data-size": "Resource files sum size",
"maximum-resources-sum-data-size-required": "Resource files sum size is required.",
"maximum-resources-sum-data-size-range": "Resource files sum size can`t be negative",
"maximum-resources-sum-data-size-range": "Resource files sum size can't be negative",
"maximum-resource-size": "Resource maximum size",
"maximum-resource-size-required": "Resource maximum size is required",
"maximum-resource-size-range": "Resource maximum size can't be negative",
"maximum-ota-packages-sum-data-size": "OTA package files sum size",
"maximum-ota-package-sum-data-size-required": "OTA package files sum size is required.",
"maximum-ota-package-sum-data-size-range": "OTA package files sum size can`t be negative",
"maximum-ota-package-sum-data-size-range": "OTA package files sum size can't be negative",
"rest-requests-for-tenant": "REST requests for tenant",
"transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages",
"transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points",

Loading…
Cancel
Save