tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

device count performance for 100k+ devices per tenant. Added additional tests for the DeviceService

pull/4470/head
Sergey Matvienko 5 years ago
committed by Andrew Shvayka
parent
91e0a77afd
commit
cb9450104e
6 changed files with 160 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java
  2. 6
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
  3. 5
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
  4. 13
      dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java
  5. 11
      dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java
  6. 183
      dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java

2
common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java
View File

@ -98,4 +98,6 @@ public interface DeviceService {
PageData<Device> findDevicesByTenantIdAndEdgeId(TenantId tenantId, EdgeId edgeId, PageLink pageLink);
PageData<Device> findDevicesByTenantIdAndEdgeIdAndType(TenantId tenantId, EdgeId edgeId, String type, PageLink pageLink);
long countByTenantId(TenantId tenantId);
}

6
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java
View File

@ -15,10 +15,16 @@
*/
package org.thingsboard.server.common.data.tenant.profile;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.ApiUsageRecordKey;
import org.thingsboard.server.common.data.TenantProfileType;
@AllArgsConstructor
@NoArgsConstructor
@Builder
@Data
public class DefaultTenantProfileConfiguration implements TenantProfileConfiguration {

5
dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java
View File

@ -615,6 +615,11 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe
return deviceDao.findDevicesByTenantIdAndEdgeIdAndType(tenantId.getId(), edgeId.getId(), type, pageLink);
}
@Override
public long countByTenantId(TenantId tenantId) {
return deviceDao.countByTenantId(tenantId);
}
private DataValidator<Device> deviceValidator =
new DataValidator<Device>() {

13
dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java
View File

@ -189,5 +189,16 @@ public interface DeviceRepository extends PagingAndSortingRepository<DeviceEntit
@Param("searchText") String searchText,
Pageable pageable);
Long countByTenantId(UUID tenantId);
/**
* Count devices by tenantId.
* Custom query applied because default QueryDSL produces slow count(id).
* <p>
* There is two way to count devices.
* OPTIMAL: count(*)
* - returns _row_count_ and use index-only scan (super fast).
* SLOW: count(id)
* - returns _NON_NULL_id_count and performs table scan to verify isNull for each id in filtered rows.
* */
@Query("SELECT count(*) FROM DeviceEntity d WHERE d.tenantId = :tenantId")
Long countByTenantId(@Param("tenantId") UUID tenantId);
}

11
dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java
View File

@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.DeviceProfileType;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
@ -70,6 +71,8 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import static org.junit.Assert.assertNotNull;
@RunWith(SpringRunner.class)
@ContextConfiguration(classes = AbstractServiceTest.class, loader = AnnotationConfigContextLoader.class)
@ -225,4 +228,12 @@ public abstract class AbstractServiceTest {
return deviceProfile;
}
public TenantId createTenant() {
Tenant tenant = new Tenant();
tenant.setTitle("My tenant " + Uuids.timeBased());
Tenant savedTenant = tenantService.saveTenant(tenant);
assertNotNull(savedTenant);
return savedTenant.getId();
}
}

183
dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java
View File

@ -21,13 +21,19 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.*;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceInfo;
import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException;
import java.util.ArrayList;
@ -37,33 +43,93 @@ import java.util.List;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
private IdComparator<Device> idComparator = new IdComparator<>();
private TenantId tenantId;
private TenantId anotherTenantId;
@Before
public void before() {
Tenant tenant = new Tenant();
tenant.setTitle("My tenant");
Tenant savedTenant = tenantService.saveTenant(tenant);
Assert.assertNotNull(savedTenant);
tenantId = savedTenant.getId();
tenantId = createTenant();
anotherTenantId = createTenant();
}
@After
public void after() {
tenantService.deleteTenant(tenantId);
tenantService.deleteTenant(anotherTenantId);
tenantProfileService.deleteTenantProfiles(tenantId);
tenantProfileService.deleteTenantProfiles(anotherTenantId);
}
@Test
public void testSaveDevicesWithoutMaxDeviceLimit() {
Device device = this.saveDevice(tenantId, "My device");
deleteDevice(tenantId, device);
}
@Test
public void testSaveDevice() {
public void testSaveDevicesWithInfiniteMaxDeviceLimit() {
TenantProfile defaultTenantProfile = tenantProfileService.findDefaultTenantProfile(tenantId);
defaultTenantProfile.getProfileData().setConfiguration(DefaultTenantProfileConfiguration.builder().maxDevices(Long.MAX_VALUE).build());
tenantProfileService.saveTenantProfile(tenantId, defaultTenantProfile);
Device device = this.saveDevice(tenantId, "My device");
deleteDevice(tenantId, device);
}
@Test(expected = DataValidationException.class)
public void testSaveDevicesWithMaxDeviceOutOfLimit() {
TenantProfile defaultTenantProfile = tenantProfileService.findDefaultTenantProfile(tenantId);
defaultTenantProfile.getProfileData().setConfiguration(DefaultTenantProfileConfiguration.builder().maxDevices(1).build());
tenantProfileService.saveTenantProfile(tenantId, defaultTenantProfile);
Assert.assertEquals(0, deviceService.countByTenantId(tenantId));
this.saveDevice(tenantId, "My first device");
Assert.assertEquals(1, deviceService.countByTenantId(tenantId));
this.saveDevice(tenantId, "My second device that out of maxDeviceCount limit");
}
@Test
public void testCountByTenantId() {
Assert.assertEquals(0, deviceService.countByTenantId(tenantId));
Assert.assertEquals(0, deviceService.countByTenantId(anotherTenantId));
Assert.assertEquals(0, deviceService.countByTenantId(TenantId.SYS_TENANT_ID));
Device anotherDevice = this.saveDevice(anotherTenantId, "My device 1");
Assert.assertEquals(1, deviceService.countByTenantId(anotherTenantId));
int maxDevices = 8;
List<Device> devices = new ArrayList<>(maxDevices);
for (int i = 1; i <= maxDevices; i++) {
devices.add(this.saveDevice(tenantId, "My device " + i));
Assert.assertEquals(i, deviceService.countByTenantId(tenantId));
}
Assert.assertEquals(maxDevices, deviceService.countByTenantId(tenantId));
Assert.assertEquals(1, deviceService.countByTenantId(anotherTenantId));
Assert.assertEquals(0, deviceService.countByTenantId(TenantId.SYS_TENANT_ID));
devices.forEach(device -> deleteDevice(tenantId, device));
deleteDevice(anotherTenantId, anotherDevice);
}
void deleteDevice(TenantId tenantId, Device device) {
deviceService.deleteDevice(tenantId, device.getId());
}
Device saveDevice(TenantId tenantId, final String name) {
Device device = new Device();
device.setTenantId(tenantId);
device.setName("My device");
device.setName(name);
device.setType("default");
Device savedDevice = deviceService.saveDevice(device);
Assert.assertNotNull(savedDevice);
Assert.assertNotNull(savedDevice.getId());
Assert.assertTrue(savedDevice.getCreatedTime() > 0);
@ -71,7 +137,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
Assert.assertNotNull(savedDevice.getCustomerId());
Assert.assertEquals(NULL_UUID, savedDevice.getCustomerId().getId());
Assert.assertEquals(device.getName(), savedDevice.getName());
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId());
Assert.assertNotNull(deviceCredentials);
Assert.assertNotNull(deviceCredentials.getId());
@ -79,16 +145,15 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
Assert.assertEquals(DeviceCredentialsType.ACCESS_TOKEN, deviceCredentials.getCredentialsType());
Assert.assertNotNull(deviceCredentials.getCredentialsId());
Assert.assertEquals(20, deviceCredentials.getCredentialsId().length());
savedDevice.setName("My new device");
savedDevice.setName("New " + savedDevice.getName());
deviceService.saveDevice(savedDevice);
Device foundDevice = deviceService.findDeviceById(tenantId, savedDevice.getId());
Assert.assertEquals(foundDevice.getName(), savedDevice.getName());
deviceService.deleteDevice(tenantId, savedDevice.getId());
return foundDevice;
}
@Test(expected = DataValidationException.class)
public void testSaveDeviceWithEmptyName() {
Device device = new Device();
@ -96,7 +161,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device.setTenantId(tenantId);
deviceService.saveDevice(device);
}
@Test(expected = DataValidationException.class)
public void testSaveDeviceWithEmptyTenant() {
Device device = new Device();
@ -104,7 +169,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device.setType("default");
deviceService.saveDevice(device);
}
@Test(expected = DataValidationException.class)
public void testSaveDeviceWithInvalidTenant() {
Device device = new Device();
@ -113,7 +178,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device.setTenantId(new TenantId(Uuids.timeBased()));
deviceService.saveDevice(device);
}
@Test(expected = DataValidationException.class)
public void testAssignDeviceToNonExistentCustomer() {
Device device = new Device();
@ -149,7 +214,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
tenantService.deleteTenant(tenant.getId());
}
}
@Test
public void testFindDeviceById() {
Device device = new Device();
@ -198,7 +263,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
devices.forEach((device) -> { deviceService.deleteDevice(tenantId, device.getId()); });
}
}
@Test
public void testDeleteDevice() {
Device device = new Device();
@ -214,15 +279,15 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
DeviceCredentials foundDeviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId());
Assert.assertNull(foundDeviceCredentials);
}
@Test
public void testFindDevicesByTenantId() {
Tenant tenant = new Tenant();
tenant.setTitle("Test tenant");
tenant = tenantService.saveTenant(tenant);
TenantId tenantId = tenant.getId();
List<Device> devices = new ArrayList<>();
for (int i=0;i<178;i++) {
Device device = new Device();
@ -231,7 +296,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device.setType("default");
devices.add(deviceService.saveDevice(device));
}
List<Device> loadedDevices = new ArrayList<>();
PageLink pageLink = new PageLink(23);
PageData<Device> pageData = null;
@ -242,19 +307,19 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
pageLink = pageLink.nextPageLink();
}
} while (pageData.hasNext());
Collections.sort(devices, idComparator);
Collections.sort(loadedDevices, idComparator);
Assert.assertEquals(devices, loadedDevices);
deviceService.deleteDevicesByTenantId(tenantId);
pageLink = new PageLink(33);
pageData = deviceService.findDevicesByTenantId(tenantId, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertTrue(pageData.getData().isEmpty());
tenantService.deleteTenant(tenantId);
}
@ -284,7 +349,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device.setType("default");
devicesTitle2.add(new DeviceInfo(deviceService.saveDevice(device), null, false, "default"));
}
List<DeviceInfo> loadedDevicesTitle1 = new ArrayList<>();
PageLink pageLink = new PageLink(15, 0, title1);
PageData<DeviceInfo> pageData = null;
@ -295,12 +360,12 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
pageLink = pageLink.nextPageLink();
}
} while (pageData.hasNext());
Collections.sort(devicesTitle1, idComparator);
Collections.sort(loadedDevicesTitle1, idComparator);
Assert.assertEquals(devicesTitle1, loadedDevicesTitle1);
List<DeviceInfo> loadedDevicesTitle2 = new ArrayList<>();
pageLink = new PageLink(4, 0, title2);
do {
@ -313,22 +378,22 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
Collections.sort(devicesTitle2, idComparator);
Collections.sort(loadedDevicesTitle2, idComparator);
Assert.assertEquals(devicesTitle2, loadedDevicesTitle2);
for (Device device : loadedDevicesTitle1) {
deviceService.deleteDevice(tenantId, device.getId());
}
pageLink = new PageLink(4, 0, title1);
pageData = deviceService.findDeviceInfosByTenantId(tenantId, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
for (Device device : loadedDevicesTitle2) {
deviceService.deleteDevice(tenantId, device.getId());
}
pageLink = new PageLink(4, 0, title2);
pageData = deviceService.findDeviceInfosByTenantId(tenantId, pageLink);
Assert.assertFalse(pageData.hasNext());
@ -413,21 +478,21 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
}
@Test
public void testFindDevicesByTenantIdAndCustomerId() {
Tenant tenant = new Tenant();
tenant.setTitle("Test tenant");
tenant = tenantService.saveTenant(tenant);
TenantId tenantId = tenant.getId();
Customer customer = new Customer();
customer.setTitle("Test customer");
customer.setTenantId(tenantId);
customer = customerService.saveCustomer(customer);
CustomerId customerId = customer.getId();
List<DeviceInfo> devices = new ArrayList<>();
for (int i=0;i<278;i++) {
Device device = new Device();
@ -437,7 +502,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device = deviceService.saveDevice(device);
devices.add(new DeviceInfo(deviceService.assignDeviceToCustomer(tenantId, device.getId(), customerId), customer.getTitle(), customer.isPublic(), "default"));
}
List<DeviceInfo> loadedDevices = new ArrayList<>();
PageLink pageLink = new PageLink(23);
PageData<DeviceInfo> pageData = null;
@ -448,31 +513,31 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
pageLink = pageLink.nextPageLink();
}
} while (pageData.hasNext());
Collections.sort(devices, idComparator);
Collections.sort(loadedDevices, idComparator);
Assert.assertEquals(devices, loadedDevices);
deviceService.unassignCustomerDevices(tenantId, customerId);
pageLink = new PageLink(33);
pageData = deviceService.findDeviceInfosByTenantIdAndCustomerId(tenantId, customerId, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertTrue(pageData.getData().isEmpty());
tenantService.deleteTenant(tenantId);
}
@Test
public void testFindDevicesByTenantIdCustomerIdAndName() {
Customer customer = new Customer();
customer.setTitle("Test customer");
customer.setTenantId(tenantId);
customer = customerService.saveCustomer(customer);
CustomerId customerId = customer.getId();
String title1 = "Device title 1";
List<Device> devicesTitle1 = new ArrayList<>();
for (int i=0;i<175;i++) {
@ -499,7 +564,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
device = deviceService.saveDevice(device);
devicesTitle2.add(deviceService.assignDeviceToCustomer(tenantId, device.getId(), customerId));
}
List<Device> loadedDevicesTitle1 = new ArrayList<>();
PageLink pageLink = new PageLink(15, 0, title1);
PageData<Device> pageData = null;
@ -510,12 +575,12 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
pageLink = pageLink.nextPageLink();
}
} while (pageData.hasNext());
Collections.sort(devicesTitle1, idComparator);
Collections.sort(loadedDevicesTitle1, idComparator);
Assert.assertEquals(devicesTitle1, loadedDevicesTitle1);
List<Device> loadedDevicesTitle2 = new ArrayList<>();
pageLink = new PageLink(4, 0, title2);
do {
@ -528,22 +593,22 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
Collections.sort(devicesTitle2, idComparator);
Collections.sort(loadedDevicesTitle2, idComparator);
Assert.assertEquals(devicesTitle2, loadedDevicesTitle2);
for (Device device : loadedDevicesTitle1) {
deviceService.deleteDevice(tenantId, device.getId());
}
pageLink = new PageLink(4, 0, title1);
pageData = deviceService.findDevicesByTenantIdAndCustomerId(tenantId, customerId, pageLink);
Assert.assertFalse(pageData.hasNext());
Assert.assertEquals(0, pageData.getData().size());
for (Device device : loadedDevicesTitle2) {
deviceService.deleteDevice(tenantId, device.getId());
}
pageLink = new PageLink(4, 0, title2);
pageData = deviceService.findDevicesByTenantIdAndCustomerId(tenantId, customerId, pageLink);
Assert.assertFalse(pageData.hasNext());

Write Preview
Loading…
Cancel
Save