Browse Source
Fixed notification requests and RPC cleanup timeout on large datasetspull/15195/merge
Viacheslav Klimov
3 months ago
committed by
GitHub
GitHub
14 changed files with 567 additions and 62 deletions
@ -0,0 +1,144 @@ |
|||
/** |
|||
* Copyright © 2016-2026 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.ttl; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.junit.jupiter.api.extension.ExtendWith; |
|||
import org.mockito.Mock; |
|||
import org.mockito.junit.jupiter.MockitoExtension; |
|||
import org.springframework.test.util.ReflectionTestUtils; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.notification.NotificationRequestDao; |
|||
import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; |
|||
import org.thingsboard.server.dao.tenant.TenantService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
|
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
import static org.mockito.ArgumentMatchers.any; |
|||
import static org.mockito.ArgumentMatchers.anyInt; |
|||
import static org.mockito.ArgumentMatchers.anyLong; |
|||
import static org.mockito.ArgumentMatchers.anyString; |
|||
import static org.mockito.ArgumentMatchers.eq; |
|||
import static org.mockito.Mockito.never; |
|||
import static org.mockito.Mockito.times; |
|||
import static org.mockito.Mockito.verify; |
|||
import static org.mockito.Mockito.when; |
|||
|
|||
@ExtendWith(MockitoExtension.class) |
|||
public class NotificationsCleanUpServiceTest { |
|||
|
|||
@Mock |
|||
private PartitionService partitionService; |
|||
@Mock |
|||
private SqlPartitioningRepository partitioningRepository; |
|||
@Mock |
|||
private NotificationRequestDao notificationRequestDao; |
|||
@Mock |
|||
private TenantService tenantService; |
|||
|
|||
private NotificationsCleanUpService cleanUpService; |
|||
|
|||
private static final int BATCH_SIZE = 3; |
|||
|
|||
@BeforeEach |
|||
public void setUp() { |
|||
cleanUpService = new NotificationsCleanUpService(partitionService, partitioningRepository, notificationRequestDao, tenantService); |
|||
ReflectionTestUtils.setField(cleanUpService, "ttlInSec", 2592000L); |
|||
ReflectionTestUtils.setField(cleanUpService, "partitionSizeInHours", 168); |
|||
ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); |
|||
} |
|||
|
|||
@Test |
|||
public void testBatchLoopCallsDaoMultipleTimes() { |
|||
TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); |
|||
when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); |
|||
when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) |
|||
.thenReturn(System.currentTimeMillis()); |
|||
|
|||
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); |
|||
when(tenantService.findTenantsIds(any())) |
|||
.thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); |
|||
|
|||
// Sysadmin: returns 3 (full batch), then 1 (partial) -> 2 calls
|
|||
when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(BATCH_SIZE) |
|||
.thenReturn(1); |
|||
// Tenant: returns 3, 3, 0 -> 3 calls
|
|||
when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(BATCH_SIZE) |
|||
.thenReturn(BATCH_SIZE) |
|||
.thenReturn(0); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(notificationRequestDao, times(2)) |
|||
.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); |
|||
verify(notificationRequestDao, times(3)) |
|||
.removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSkipsTenantNotOnMyPartition() { |
|||
TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); |
|||
TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); |
|||
when(partitionService.resolve(any(), eq(TenantId.SYS_TENANT_ID), eq(TenantId.SYS_TENANT_ID))) |
|||
.thenReturn(myPartition); |
|||
when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) |
|||
.thenReturn(System.currentTimeMillis()); |
|||
|
|||
// Sysadmin: no records
|
|||
when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(0); |
|||
|
|||
TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); |
|||
TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); |
|||
when(tenantService.findTenantsIds(any())) |
|||
.thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); |
|||
when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); |
|||
when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); |
|||
|
|||
when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(0); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); |
|||
verify(notificationRequestDao, never()).removeByTenantIdAndCreatedTimeBeforeBatch(eq(otherTenant), anyLong(), anyInt()); |
|||
} |
|||
|
|||
@Test |
|||
public void testNoPartitionsDropped_stillCleansUpRequests() { |
|||
TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); |
|||
when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); |
|||
when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) |
|||
.thenReturn(0L); |
|||
|
|||
when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(0); |
|||
when(tenantService.findTenantsIds(any())) |
|||
.thenReturn(new PageData<>(List.of(), 0, 0, false)); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,136 @@ |
|||
/** |
|||
* Copyright © 2016-2026 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.ttl.rpc; |
|||
|
|||
import org.junit.jupiter.api.BeforeEach; |
|||
import org.junit.jupiter.api.Test; |
|||
import org.junit.jupiter.api.extension.ExtendWith; |
|||
import org.mockito.Mock; |
|||
import org.mockito.junit.jupiter.MockitoExtension; |
|||
import org.springframework.test.util.ReflectionTestUtils; |
|||
import org.thingsboard.server.common.data.TenantProfile; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.page.PageData; |
|||
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; |
|||
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; |
|||
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; |
|||
import org.thingsboard.server.dao.rpc.RpcDao; |
|||
import org.thingsboard.server.dao.tenant.TbTenantProfileCache; |
|||
import org.thingsboard.server.dao.tenant.TenantService; |
|||
import org.thingsboard.server.queue.discovery.PartitionService; |
|||
|
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
|
|||
import static org.mockito.ArgumentMatchers.any; |
|||
import static org.mockito.ArgumentMatchers.anyInt; |
|||
import static org.mockito.ArgumentMatchers.anyLong; |
|||
import static org.mockito.ArgumentMatchers.eq; |
|||
import static org.mockito.Mockito.never; |
|||
import static org.mockito.Mockito.times; |
|||
import static org.mockito.Mockito.verify; |
|||
import static org.mockito.Mockito.when; |
|||
|
|||
@ExtendWith(MockitoExtension.class) |
|||
public class RpcCleanUpServiceTest { |
|||
|
|||
@Mock |
|||
private PartitionService partitionService; |
|||
@Mock |
|||
private RpcDao rpcDao; |
|||
@Mock |
|||
private TenantService tenantService; |
|||
@Mock |
|||
private TbTenantProfileCache tenantProfileCache; |
|||
|
|||
private RpcCleanUpService cleanUpService; |
|||
|
|||
private static final int BATCH_SIZE = 3; |
|||
|
|||
@BeforeEach |
|||
public void setUp() { |
|||
cleanUpService = new RpcCleanUpService(tenantService, partitionService, tenantProfileCache, rpcDao); |
|||
ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); |
|||
} |
|||
|
|||
@Test |
|||
public void testBatchLoopCallsDaoMultipleTimes() { |
|||
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); |
|||
setupTenant(tenantId, 7); |
|||
|
|||
// Returns 3 (full batch), 3 (full batch), 1 (partial) -> 3 calls
|
|||
when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(BATCH_SIZE) |
|||
.thenReturn(BATCH_SIZE) |
|||
.thenReturn(1); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(rpcDao, times(3)).deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); |
|||
} |
|||
|
|||
@Test |
|||
public void testSkipsTenantNotOnMyPartition() { |
|||
TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); |
|||
TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); |
|||
|
|||
TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); |
|||
TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); |
|||
|
|||
when(tenantService.findTenantsIds(any())) |
|||
.thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); |
|||
when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); |
|||
when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); |
|||
|
|||
setupTenantProfile(myTenant, 7); |
|||
when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) |
|||
.thenReturn(0); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(rpcDao).deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); |
|||
verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(eq(otherTenant), anyLong(), anyInt()); |
|||
} |
|||
|
|||
@Test |
|||
public void testSkipsTenantWithZeroTtl() { |
|||
TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); |
|||
setupTenant(tenantId, 0); |
|||
|
|||
cleanUpService.cleanUp(); |
|||
|
|||
verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(any(), anyLong(), anyInt()); |
|||
} |
|||
|
|||
private void setupTenant(TenantId tenantId, int rpcTtlDays) { |
|||
TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); |
|||
when(partitionService.resolve(any(), eq(tenantId), eq(tenantId))).thenReturn(myPartition); |
|||
when(tenantService.findTenantsIds(any())) |
|||
.thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); |
|||
setupTenantProfile(tenantId, rpcTtlDays); |
|||
} |
|||
|
|||
private void setupTenantProfile(TenantId tenantId, int rpcTtlDays) { |
|||
TenantProfile profile = new TenantProfile(); |
|||
TenantProfileData profileData = new TenantProfileData(); |
|||
DefaultTenantProfileConfiguration config = new DefaultTenantProfileConfiguration(); |
|||
config.setRpcTtlDays(rpcTtlDays); |
|||
profileData.setConfiguration(config); |
|||
profile.setProfileData(profileData); |
|||
when(tenantProfileCache.get(tenantId)).thenReturn(profile); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,133 @@ |
|||
/** |
|||
* Copyright © 2016-2026 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.dao.sql.notification; |
|||
|
|||
import org.junit.After; |
|||
import org.junit.Test; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.thingsboard.server.common.data.id.NotificationRequestId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.common.data.notification.NotificationRequest; |
|||
import org.thingsboard.server.common.data.notification.NotificationRequestStatus; |
|||
import org.thingsboard.server.dao.AbstractJpaDaoTest; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.List; |
|||
import java.util.UUID; |
|||
import java.util.concurrent.TimeUnit; |
|||
|
|||
import static org.assertj.core.api.Assertions.assertThat; |
|||
|
|||
public class JpaNotificationRequestDaoTest extends AbstractJpaDaoTest { |
|||
|
|||
@Autowired |
|||
JpaNotificationRequestDao notificationRequestDao; |
|||
|
|||
private final List<NotificationRequest> createdRequests = new ArrayList<>(); |
|||
|
|||
@After |
|||
public void tearDown() { |
|||
for (NotificationRequest request : createdRequests) { |
|||
notificationRequestDao.removeById(request.getTenantId(), request.getId().getId()); |
|||
} |
|||
createdRequests.clear(); |
|||
} |
|||
|
|||
@Test |
|||
public void testBatchDeletion() { |
|||
TenantId sysTenantId = TenantId.SYS_TENANT_ID; |
|||
long now = System.currentTimeMillis(); |
|||
long oldTimestamp = now - TimeUnit.DAYS.toMillis(30); |
|||
|
|||
NotificationRequest oldRequest1 = createNotificationRequest(sysTenantId, oldTimestamp); |
|||
notificationRequestDao.save(sysTenantId, oldRequest1); |
|||
|
|||
NotificationRequest oldRequest2 = createNotificationRequest(sysTenantId, oldTimestamp); |
|||
notificationRequestDao.save(sysTenantId, oldRequest2); |
|||
|
|||
NotificationRequest freshRequest = createNotificationRequest(sysTenantId, now); |
|||
notificationRequestDao.save(sysTenantId, freshRequest); |
|||
|
|||
TenantId tenant2Id = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); |
|||
NotificationRequest tenant2Request = createNotificationRequest(tenant2Id, oldTimestamp); |
|||
notificationRequestDao.save(tenant2Id, tenant2Request); |
|||
|
|||
int batchSize = 10_000; |
|||
|
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, oldTimestamp - 1, batchSize)).isEqualTo(0); |
|||
|
|||
long expirationTime = now - TimeUnit.DAYS.toMillis(15); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, expirationTime, batchSize)).isEqualTo(2); |
|||
|
|||
assertThat(notificationRequestDao.findById(sysTenantId, freshRequest.getId().getId())).isNotNull(); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2Id, now + 1, batchSize)).isEqualTo(1); |
|||
} |
|||
|
|||
@Test |
|||
public void testBatchDeletionWithSmallBatchSize() { |
|||
TenantId tenantId = TenantId.SYS_TENANT_ID; |
|||
long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); |
|||
|
|||
for (int i = 0; i < 10; i++) { |
|||
NotificationRequest request = createNotificationRequest(tenantId, oldTimestamp); |
|||
notificationRequestDao.save(tenantId, request); |
|||
} |
|||
|
|||
int batchSize = 3; |
|||
long expirationTime = System.currentTimeMillis(); |
|||
|
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(1); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(0); |
|||
} |
|||
|
|||
@Test |
|||
public void testBatchDeletionIsolationBetweenTenants() { |
|||
TenantId tenant1 = TenantId.SYS_TENANT_ID; |
|||
TenantId tenant2 = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); |
|||
long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); |
|||
|
|||
for (int i = 0; i < 5; i++) { |
|||
NotificationRequest request = createNotificationRequest(tenant1, oldTimestamp); |
|||
notificationRequestDao.save(tenant1, request); |
|||
} |
|||
|
|||
for (int i = 0; i < 3; i++) { |
|||
NotificationRequest request = createNotificationRequest(tenant2, oldTimestamp); |
|||
notificationRequestDao.save(tenant2, request); |
|||
} |
|||
|
|||
int batchSize = 10_000; |
|||
long expirationTime = System.currentTimeMillis(); |
|||
|
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant1, expirationTime, batchSize)).isEqualTo(5); |
|||
assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2, expirationTime, batchSize)).isEqualTo(3); |
|||
} |
|||
|
|||
private NotificationRequest createNotificationRequest(TenantId tenantId, long createdTime) { |
|||
NotificationRequest request = new NotificationRequest(); |
|||
request.setId(new NotificationRequestId(UUID.randomUUID())); |
|||
request.setTenantId(tenantId); |
|||
request.setCreatedTime(createdTime); |
|||
request.setTargets(List.of(UUID.randomUUID())); |
|||
request.setStatus(NotificationRequestStatus.SENT); |
|||
createdRequests.add(request); |
|||
return request; |
|||
} |
|||
|
|||
} |
|||
Loading…
Reference in new issue