diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index 5cf051367a..b428e1cc05 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -16,8 +16,6 @@ package org.thingsboard.server.service.resource; import lombok.extern.slf4j.Slf4j; -import org.eclipse.leshan.core.model.DDFFileParser; -import org.eclipse.leshan.core.model.DefaultDDFFileValidator; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; @@ -53,11 +51,9 @@ import static org.thingsboard.server.utils.LwM2mObjectModelUtils.toLwm2mResource public class DefaultTbResourceService extends AbstractTbEntityService implements TbResourceService { private final ResourceService resourceService; - private final DDFFileParser ddfFileParser; public DefaultTbResourceService(ResourceService resourceService) { this.resourceService = resourceService; - this.ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java b/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java index bb2e49bd62..4a07444bbb 100644 --- a/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java +++ b/application/src/main/java/org/thingsboard/server/utils/LwM2mObjectModelUtils.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.lwm2m.LwM2mInstance; import org.thingsboard.server.common.data.lwm2m.LwM2mObject; import org.thingsboard.server.common.data.lwm2m.LwM2mResourceObserve; +import org.thingsboard.server.common.data.util.TbDDFFileParser; import org.thingsboard.server.dao.exception.DataValidationException; import java.io.ByteArrayInputStream; @@ -41,7 +42,7 @@ import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPA @Slf4j public class LwM2mObjectModelUtils { - private static final DDFFileParser ddfFileParser = new DDFFileParser(new DefaultDDFFileValidator()); + private static final TbDDFFileParser ddfFileParser = new TbDDFFileParser(); public static void toLwm2mResource (TbResource resource) throws ThingsboardException { try { diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index e2c5985553..5c9d94cde2 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -45,6 +45,8 @@ import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -75,6 +77,32 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { "\n" + ""; + private static final String LWM2M_TEST_MODEL_WITH_XXE = " ]>" + + "\n" + + "\n" + + "My first resource\n" + + "\n" + + "0\n" + + "\n" + + "&ObjectVersion;\n" + + "Multiple\n" + + "Mandatory\n" + + "\n" + + "\n" + + "LWM2M\n" + + "RW\n" + + "Single\n" + + "Mandatory\n" + + "String\n" + + "0..255\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + ""; + private static final String DEFAULT_FILE_NAME = "test.jks"; private IdComparator idComparator = new IdComparator<>(); @@ -126,11 +154,11 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { loginTenantAdmin(); - Assert.assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId)); createResource("test", DEFAULT_FILE_NAME); - Assert.assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId)); try { assertThatThrownBy(() -> createResource("test1", 1 + DEFAULT_FILE_NAME)) @@ -145,19 +173,19 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { @Test public void sumDataSizeByTenantId() throws Exception { - Assert.assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(0, resourceService.sumDataSizeByTenantId(tenantId)); createResource("test", DEFAULT_FILE_NAME); - Assert.assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(1, resourceService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; for (int i = 2; i <= maxSumDataSize; i++) { createResource("test" + i, i + DEFAULT_FILE_NAME); - Assert.assertEquals(i, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(i, resourceService.sumDataSizeByTenantId(tenantId)); } - Assert.assertEquals(maxSumDataSize, resourceService.sumDataSizeByTenantId(tenantId)); + assertEquals(maxSumDataSize, resourceService.sumDataSizeByTenantId(tenantId)); } private TbResource createResource(String title, String filename) throws Exception { @@ -184,16 +212,16 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(savedResource); Assert.assertNotNull(savedResource.getId()); Assert.assertTrue(savedResource.getCreatedTime() > 0); - Assert.assertEquals(resource.getTenantId(), savedResource.getTenantId()); - Assert.assertEquals(resource.getTitle(), savedResource.getTitle()); - Assert.assertEquals(resource.getResourceKey(), savedResource.getResourceKey()); - Assert.assertEquals(resource.getData(), savedResource.getData()); + assertEquals(resource.getTenantId(), savedResource.getTenantId()); + assertEquals(resource.getTitle(), savedResource.getTitle()); + assertEquals(resource.getResourceKey(), savedResource.getResourceKey()); + assertEquals(resource.getData(), savedResource.getData()); savedResource.setTitle("My new resource"); resourceService.save(savedResource); TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); - Assert.assertEquals(foundResource.getTitle(), savedResource.getTitle()); + assertEquals(foundResource.getTitle(), savedResource.getTitle()); resourceService.delete(savedResource, null); } @@ -211,10 +239,10 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertNotNull(savedResource); Assert.assertNotNull(savedResource.getId()); Assert.assertTrue(savedResource.getCreatedTime() > 0); - Assert.assertEquals(resource.getTenantId(), savedResource.getTenantId()); - Assert.assertEquals("My first resource id=0 v1.0", savedResource.getTitle()); - Assert.assertEquals("0_1.0", savedResource.getResourceKey()); - Assert.assertEquals(resource.getData(), savedResource.getData()); + assertEquals(resource.getTenantId(), savedResource.getTenantId()); + assertEquals("My first resource id=0 v1.0", savedResource.getTitle()); + assertEquals("0_1.0", savedResource.getResourceKey()); + assertEquals(resource.getData(), savedResource.getData()); resourceService.delete(savedResource, null); } @@ -228,7 +256,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { resource.setData("Test Data"); TbResource savedResource = resourceService.save(resource); - Assert.assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); + assertEquals(TenantId.SYS_TENANT_ID, savedResource.getTenantId()); resourceService.delete(savedResource, null); } @@ -285,6 +313,21 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { }); } + @Test + public void testSaveLwm2mTbResourceWithXXE() { + TbResource resource = new TbResource(); + resource.setTenantId(tenantId); + resource.setResourceType(ResourceType.LWM2M_MODEL); + resource.setFileName("xxe_test_model.xml"); + resource.setData(Base64.getEncoder().encodeToString(LWM2M_TEST_MODEL_WITH_XXE.getBytes())); + + DataValidationException thrown = assertThrows(DataValidationException.class, () -> { + resourceService.save(resource); + }); + assertEquals("Failed to parse file xxe_test_model.xml", thrown.getMessage()); + } + + @Test public void testFindResourceById() throws Exception { TbResource resource = new TbResource(); @@ -296,7 +339,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { TbResource foundResource = resourceService.findResourceById(tenantId, savedResource.getId()); Assert.assertNotNull(foundResource); - Assert.assertEquals(savedResource, foundResource); + assertEquals(savedResource, foundResource); resourceService.delete(savedResource, null); } @@ -312,7 +355,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { TbResource foundResource = resourceService.getResource(tenantId, savedResource.getResourceType(), savedResource.getResourceKey()); Assert.assertNotNull(foundResource); - Assert.assertEquals(savedResource, foundResource); + assertEquals(savedResource, foundResource); resourceService.delete(savedResource, null); } @@ -366,7 +409,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Collections.sort(resources, idComparator); Collections.sort(loadedResources, idComparator); - Assert.assertEquals(resources, loadedResources); + assertEquals(resources, loadedResources); resourceService.deleteResourcesByTenantId(tenantId); @@ -427,14 +470,14 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Collections.sort(resources, idComparator); Collections.sort(loadedResources, idComparator); - Assert.assertEquals(resources, loadedResources); + assertEquals(resources, loadedResources); resourceService.deleteResourcesByTenantId(tenantId); pageLink = new PageLink(100); pageData = resourceService.findAllTenantResourcesByTenantId(tenantId, pageLink); Assert.assertFalse(pageData.hasNext()); - Assert.assertEquals(pageData.getData().size(), 100); + assertEquals(pageData.getData().size(), 100); resourceService.deleteResourcesByTenantId(TenantId.SYS_TENANT_ID); diff --git a/common/data/pom.xml b/common/data/pom.xml index d81203cd4d..ba977c1318 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -116,6 +116,11 @@ com.google.protobuf protobuf-java-util + + org.eclipse.leshan + leshan-core + compile + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/TbDDFFileParser.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbDDFFileParser.java new file mode 100644 index 0000000000..7c1b9307c5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/TbDDFFileParser.java @@ -0,0 +1,272 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.util; + +import lombok.extern.slf4j.Slf4j; +import org.eclipse.leshan.core.LwM2m; +import org.eclipse.leshan.core.model.DDFFileValidator; +import org.eclipse.leshan.core.model.DefaultDDFFileValidator; +import org.eclipse.leshan.core.model.InvalidDDFFileException; +import org.eclipse.leshan.core.model.ObjectModel; +import org.eclipse.leshan.core.model.ResourceModel; +import org.eclipse.leshan.core.util.StringUtils; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +public class TbDDFFileParser { + private static final DDFFileValidator ddfFileValidator = new DefaultDDFFileValidator(); + + public List parse(InputStream inputStream, String streamName) + throws InvalidDDFFileException, IOException { + streamName = streamName == null ? "" : streamName; + + log.debug("Parsing DDF file {}", streamName); + + try { + // Parse XML file + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setNamespaceAware(true); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(inputStream); + + // Get DDF file validator + LwM2m.LwM2mVersion lwm2mVersion = null; + ddfFileValidator.validate(document); + + // Build list of ObjectModel + ArrayList objects = new ArrayList<>(); + NodeList nodeList = document.getDocumentElement().getElementsByTagName("Object"); + for (int i = 0; i < nodeList.getLength(); i++) { + objects.add(parseObject(nodeList.item(i), streamName, lwm2mVersion, true)); + } + return objects; + } catch (InvalidDDFFileException | SAXException e) { + throw new InvalidDDFFileException(e, "Invalid DDF file %s", streamName); + } + catch (ParserConfigurationException e) { + throw new IllegalStateException("Unable to create Document Builder", e); + } + } + + private ObjectModel parseObject(Node object, String streamName, LwM2m.LwM2mVersion schemaVersion, boolean validate) + throws InvalidDDFFileException { + + Node objectType = object.getAttributes().getNamedItem("ObjectType"); + if (validate && (objectType == null || !"MODefinition".equals(objectType.getTextContent()))) { + throw new InvalidDDFFileException( + "Object element in %s MUST have a ObjectType attribute equals to 'MODefinition'.", streamName); + } + + Integer id = null; + String name = null; + String description = null; + String version = ObjectModel.DEFAULT_VERSION; + Boolean multiple = null; + Boolean mandatory = null; + Map resources = new HashMap<>(); + String urn = null; + String description2 = null; + String lwm2mVersion = ObjectModel.DEFAULT_VERSION; + + for (int i = 0; i < object.getChildNodes().getLength(); i++) { + Node field = object.getChildNodes().item(i); + if (field.getNodeType() != Node.ELEMENT_NODE) + continue; + + switch (field.getNodeName()) { + case "ObjectID": + id = Integer.valueOf(field.getTextContent()); + break; + case "Name": + name = field.getTextContent(); + break; + case "Description1": + description = field.getTextContent(); + break; + case "ObjectVersion": + if (!StringUtils.isEmpty(field.getTextContent())) { + version = field.getTextContent(); + } + break; + case "MultipleInstances": + if ("Multiple".equals(field.getTextContent())) { + multiple = true; + } else if ("Single".equals(field.getTextContent())) { + multiple = false; + } + break; + case "Mandatory": + if ("Mandatory".equals(field.getTextContent())) { + mandatory = true; + } else if ("Optional".equals(field.getTextContent())) { + mandatory = false; + } + break; + case "Resources": + for (int j = 0; j < field.getChildNodes().getLength(); j++) { + Node item = field.getChildNodes().item(j); + if (item.getNodeType() != Node.ELEMENT_NODE) + continue; + + if (item.getNodeName().equals("Item")) { + ResourceModel resource = parseResource(item, streamName); + if (validate && resources.containsKey(resource.id)) { + throw new InvalidDDFFileException( + "Object %s in %s contains at least 2 resources with same id %s.", + id != null ? id : "", streamName, resource.id); + } else { + resources.put(resource.id, resource); + } + } + } + break; + case "ObjectURN": + urn = field.getTextContent(); + break; + case "LWM2MVersion": + if (!StringUtils.isEmpty(field.getTextContent())) { + lwm2mVersion = field.getTextContent(); + if (schemaVersion != null && !schemaVersion.toString().equals(lwm2mVersion)) { + throw new InvalidDDFFileException( + "LWM2MVersion is not consistent with xml shema(xsi:noNamespaceSchemaLocation) in %s : %s expected but was %s.", + streamName, schemaVersion, lwm2mVersion); + } + } + break; + case "Description2": + description2 = field.getTextContent(); + break; + default: + break; + } + } + + return new ObjectModel(id, name, description, version, multiple, mandatory, resources.values(), urn, + lwm2mVersion, description2); + + } + + private ResourceModel parseResource(Node item, String streamName) throws DOMException, InvalidDDFFileException { + + Integer id = Integer.valueOf(item.getAttributes().getNamedItem("ID").getTextContent()); + String name = null; + ResourceModel.Operations operations = null; + Boolean multiple = false; + Boolean mandatory = false; + ResourceModel.Type type = null; + String rangeEnumeration = null; + String units = null; + String description = null; + + for (int i = 0; i < item.getChildNodes().getLength(); i++) { + Node field = item.getChildNodes().item(i); + if (field.getNodeType() != Node.ELEMENT_NODE) + continue; + + switch (field.getNodeName()) { + case "Name": + name = field.getTextContent(); + break; + case "Operations": + String strOp = field.getTextContent(); + if (strOp != null && !strOp.isEmpty()) { + operations = ResourceModel.Operations.valueOf(strOp); + } else { + operations = ResourceModel.Operations.NONE; + } + break; + case "MultipleInstances": + if ("Multiple".equals(field.getTextContent())) { + multiple = true; + } else if ("Single".equals(field.getTextContent())) { + multiple = false; + } + break; + case "Mandatory": + if ("Mandatory".equals(field.getTextContent())) { + mandatory = true; + } else if ("Optional".equals(field.getTextContent())) { + mandatory = false; + } + break; + case "Type": + switch (field.getTextContent()) { + case "String": + type = ResourceModel.Type.STRING; + break; + case "Integer": + type = ResourceModel.Type.INTEGER; + break; + case "Float": + type = ResourceModel.Type.FLOAT; + break; + case "Boolean": + type = ResourceModel.Type.BOOLEAN; + break; + case "Opaque": + type = ResourceModel.Type.OPAQUE; + break; + case "Time": + type = ResourceModel.Type.TIME; + break; + case "Objlnk": + type = ResourceModel.Type.OBJLNK; + break; + case "Unsigned Integer": + type = ResourceModel.Type.UNSIGNED_INTEGER; + break; + case "Corelnk": + type = ResourceModel.Type.CORELINK; + break; + case "": + type = ResourceModel.Type.NONE; + break; + default: + break; + } + break; + case "RangeEnumeration": + rangeEnumeration = field.getTextContent(); + break; + case "Units": + units = field.getTextContent(); + break; + case "Description": + description = field.getTextContent(); + break; + default: + break; + } + } + return new ResourceModel(id, name, operations, multiple, mandatory, type, rangeEnumeration, units, description); + } +} \ No newline at end of file diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java index 8c65c04939..ff2464f6d2 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerHelper.java @@ -25,6 +25,7 @@ import org.eclipse.leshan.core.model.ObjectModel; import org.eclipse.leshan.core.model.ResourceModel; import org.eclipse.leshan.core.node.codec.CodecException; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.util.TbDDFFileParser; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; @@ -142,9 +143,9 @@ public class LwM2mTransportServerHelper { .build(); } - public ObjectModel parseFromXmlToObjectModel(byte[] xmlByte, String streamName, DefaultDDFFileValidator ddfValidator) { + public ObjectModel parseFromXmlToObjectModel(byte[] xmlByte, String streamName) { try { - DDFFileParser ddfFileParser = new DDFFileParser(ddfValidator); + TbDDFFileParser ddfFileParser = new TbDDFFileParser(); return ddfFileParser.parse(new ByteArrayInputStream(xmlByte), streamName).get(0); } catch (IOException | InvalidDDFFileException e) { log.error("Could not parse the XML file [{}]", streamName, e); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java index d649c06b07..3dd13cd171 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mVersionedModelProvider.java @@ -154,8 +154,7 @@ public class LwM2mVersionedModelProvider implements LwM2mModelProvider { Optional tbResource = context.getTransportResourceCache().get(this.tenantId, LWM2M_MODEL, key); return tbResource.map(resource -> helper.parseFromXmlToObjectModel( Base64.getDecoder().decode(resource.getData()), - key + ".xml", - new DefaultDDFFileValidator())).orElse(null); + key + ".xml")).orElse(null); } } }