diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java index e6c3cab1ba..30bedf4eb6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitService.java @@ -119,11 +119,13 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi @Override public void update(TenantId tenantId) { - EntityTransportRateLimits tenantRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), TENANT_LIMITS); - EntityTransportRateLimits deviceRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), DEVICE_LIMITS); - EntityTransportRateLimits gatewayRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), GATEWAY_LIMITS); - EntityTransportRateLimits gatewayDeviceRateLimitPrototype = createRateLimits(tenantProfileCache.get(tenantId), GATEWAY_DEVICE_LIMITS); - update(tenantId, tenantRateLimitPrototype, deviceRateLimitPrototype, gatewayRateLimitPrototype, gatewayDeviceRateLimitPrototype); + TenantProfile profile = tenantProfileCache.get(tenantId); + update(tenantId, + createRateLimits(profile, TENANT_LIMITS), + createRateLimits(profile, DEVICE_LIMITS), + createRateLimits(profile, GATEWAY_LIMITS), + createRateLimits(profile, GATEWAY_DEVICE_LIMITS) + ); } private void update(TenantId tenantId, EntityTransportRateLimits tenantRateLimitPrototype, EntityTransportRateLimits deviceRateLimitPrototype, @@ -231,25 +233,26 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi BiConsumer putFunction) { EntityTransportRateLimits oldRateLimits = getFunction.apply(entityId); if (oldRateLimits == null) { - if (EntityType.TENANT.equals(entityId.getEntityType())) { - log.info("[{}] New rate limits: {}", entityId, newRateLimits); - } else { - log.debug("[{}] New rate limits: {}", entityId, newRateLimits); - } + logLimits(entityId, "New", newRateLimits); putFunction.accept(entityId, newRateLimits); } else { EntityTransportRateLimits updated = merge(oldRateLimits, newRateLimits); if (updated != null) { - if (EntityType.TENANT.equals(entityId.getEntityType())) { - log.info("[{}] Updated rate limits: {}", entityId, updated); - } else { - log.debug("[{}] Updated rate limits: {}", entityId, updated); - } + logLimits(entityId, "Updated", updated); putFunction.accept(entityId, updated); } } } + private void logLimits(EntityId entityId, String action, EntityTransportRateLimits limits) { + // Tenant-level changes are logged at INFO; the much noisier per-device/gateway ones at DEBUG. + if (EntityType.TENANT.equals(entityId.getEntityType())) { + log.info("[{}] {} rate limits: {}", entityId, action, limits); + } else { + log.debug("[{}] {} rate limits: {}", entityId, action, limits); + } + } + private EntityTransportRateLimits merge(EntityTransportRateLimits oldRateLimits, EntityTransportRateLimits newRateLimits) { boolean regularUpdate = !oldRateLimits.getRegularMsgRateLimit().getConfiguration().equals(newRateLimits.getRegularMsgRateLimit().getConfiguration()); boolean telemetryMsgRateUpdate = !oldRateLimits.getTelemetryMsgRateLimit().getConfiguration().equals(newRateLimits.getTelemetryMsgRateLimit().getConfiguration()); @@ -269,36 +272,12 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi DefaultTenantProfileConfiguration profile = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); if (profile == null) { return new EntityTransportRateLimits(ALLOW, ALLOW, ALLOW); - } else { - TransportRateLimit regularMsgRateLimit; - TransportRateLimit telemetryMsgRateLimit; - TransportRateLimit telemetryDpRateLimit; - switch (limitsType) { - case TENANT_LIMITS -> { - regularMsgRateLimit = newLimit(profile.getTransportTenantMsgRateLimit()); - telemetryMsgRateLimit = newLimit(profile.getTransportTenantTelemetryMsgRateLimit()); - telemetryDpRateLimit = newLimit(profile.getTransportTenantTelemetryDataPointsRateLimit()); - } - case DEVICE_LIMITS -> { - regularMsgRateLimit = newLimit(profile.getTransportDeviceMsgRateLimit()); - telemetryMsgRateLimit = newLimit(profile.getTransportDeviceTelemetryMsgRateLimit()); - telemetryDpRateLimit = newLimit(profile.getTransportDeviceTelemetryDataPointsRateLimit()); - } - case GATEWAY_LIMITS -> { - regularMsgRateLimit = newLimit(profile.getTransportGatewayMsgRateLimit()); - telemetryMsgRateLimit = newLimit(profile.getTransportGatewayTelemetryMsgRateLimit()); - telemetryDpRateLimit = newLimit(profile.getTransportGatewayTelemetryDataPointsRateLimit()); - } - case GATEWAY_DEVICE_LIMITS -> { - regularMsgRateLimit = newLimit(profile.getTransportGatewayDeviceMsgRateLimit()); - telemetryMsgRateLimit = newLimit(profile.getTransportGatewayDeviceTelemetryMsgRateLimit()); - telemetryDpRateLimit = newLimit(profile.getTransportGatewayDeviceTelemetryDataPointsRateLimit()); - } - default -> throw new IllegalStateException("Unknown limits type: " + limitsType); - } - - return new EntityTransportRateLimits(regularMsgRateLimit, telemetryMsgRateLimit, telemetryDpRateLimit); } + return new EntityTransportRateLimits( + newLimit(limitsType.getRegularMsgRateLimit().apply(profile)), + newLimit(limitsType.getTelemetryMsgRateLimit().apply(profile)), + newLimit(limitsType.getTelemetryDataPointsRateLimit().apply(profile)) + ); } private static TransportRateLimit newLimit(String config) { @@ -306,31 +285,34 @@ public class DefaultTransportRateLimitService implements TransportRateLimitServi } private EntityTransportRateLimits getTenantRateLimits(TenantId tenantId) { - return perTenantLimits.computeIfAbsent(tenantId, k -> createRateLimits(tenantProfileCache.get(tenantId), TENANT_LIMITS)); + return getRateLimits(perTenantLimits, tenantId, tenantId, TENANT_LIMITS, null); } private EntityTransportRateLimits getDeviceRateLimits(TenantId tenantId, DeviceId deviceId) { - return perDeviceLimits.computeIfAbsent(deviceId, k -> { - EntityTransportRateLimits limits = createRateLimits(tenantProfileCache.get(tenantId), DEVICE_LIMITS); - getTenantDevices(tenantId).add(deviceId); - return limits; - }); + return getRateLimits(perDeviceLimits, tenantId, deviceId, DEVICE_LIMITS, () -> getTenantDevices(tenantId).add(deviceId)); } private EntityTransportRateLimits getGatewayRateLimits(TenantId tenantId, DeviceId gatewayId) { - return perGatewayLimits.computeIfAbsent(gatewayId, k -> { - EntityTransportRateLimits limits = createRateLimits(tenantProfileCache.get(tenantId), GATEWAY_LIMITS); - getTenantGateways(tenantId).add(gatewayId); - return limits; - }); + return getRateLimits(perGatewayLimits, tenantId, gatewayId, GATEWAY_LIMITS, () -> getTenantGateways(tenantId).add(gatewayId)); } private EntityTransportRateLimits getGatewayDeviceRateLimits(TenantId tenantId, DeviceId gatewayId) { - return perGatewayDeviceLimits.computeIfAbsent(gatewayId, k -> { - EntityTransportRateLimits limits = createRateLimits(tenantProfileCache.get(tenantId), GATEWAY_DEVICE_LIMITS); - getTenantGatewayDevices(tenantId).add(gatewayId); - return limits; - }); + return getRateLimits(perGatewayDeviceLimits, tenantId, gatewayId, GATEWAY_DEVICE_LIMITS, () -> getTenantGatewayDevices(tenantId).add(gatewayId)); + } + + private EntityTransportRateLimits getRateLimits(ConcurrentMap limitsMap, TenantId tenantId, + T entityId, TransportLimitsType limitsType, Runnable onCreate) { + EntityTransportRateLimits limits = limitsMap.get(entityId); + if (limits == null) { + // Resolve the tenant profile WITHOUT holding the ConcurrentHashMap bin lock: the fetch may + // block on a cross-service round-trip, so it must run before computeIfAbsent's mapping function. + TenantProfile tenantProfile = tenantProfileCache.get(tenantId); + limits = limitsMap.computeIfAbsent(entityId, k -> createRateLimits(tenantProfile, limitsType)); + if (onCreate != null) { + onCreate.run(); + } + } + return limits; } private Set getTenantDevices(TenantId tenantId) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportLimitsType.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportLimitsType.java index 29077877a8..3a124da2b6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportLimitsType.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/limits/TransportLimitsType.java @@ -15,6 +15,39 @@ */ package org.thingsboard.server.common.transport.limits; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; + +import java.util.function.Function; + +@Getter +@RequiredArgsConstructor public enum TransportLimitsType { - TENANT_LIMITS, DEVICE_LIMITS, GATEWAY_LIMITS, GATEWAY_DEVICE_LIMITS + + TENANT_LIMITS( + DefaultTenantProfileConfiguration::getTransportTenantMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportTenantTelemetryMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportTenantTelemetryDataPointsRateLimit + ), + DEVICE_LIMITS( + DefaultTenantProfileConfiguration::getTransportDeviceMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportDeviceTelemetryMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportDeviceTelemetryDataPointsRateLimit + ), + GATEWAY_LIMITS( + DefaultTenantProfileConfiguration::getTransportGatewayMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportGatewayTelemetryMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportGatewayTelemetryDataPointsRateLimit + ), + GATEWAY_DEVICE_LIMITS( + DefaultTenantProfileConfiguration::getTransportGatewayDeviceMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportGatewayDeviceTelemetryMsgRateLimit, + DefaultTenantProfileConfiguration::getTransportGatewayDeviceTelemetryDataPointsRateLimit + ); + + private final Function regularMsgRateLimit; + private final Function telemetryMsgRateLimit; + private final Function telemetryDataPointsRateLimit; + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4923bcd9a2..8eb8e795c1 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.transport.service; +import com.google.common.util.concurrent.Striped; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -37,14 +38,15 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; @Component @TbTransportComponent @Slf4j public class DefaultTransportTenantProfileCache implements TransportTenantProfileCache { - private final Lock tenantProfileFetchLock = new ReentrantLock(); + // Bounded set of per-tenant locks: de-duplicates concurrent misses for the same tenant while + // letting different tenants fetch concurrently (eager array - no weak-ref overhead at this size). + private final Striped tenantProfileFetchLocks = Striped.lock(1024); private final ConcurrentMap profiles = new ConcurrentHashMap<>(); private final ConcurrentMap tenantIds = new ConcurrentHashMap<>(); private final ConcurrentMap> tenantProfileIds = new ConcurrentHashMap<>(); @@ -103,43 +105,53 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil } private TenantProfile getTenantProfile(TenantId tenantId) { - TenantProfile profile = null; - TenantProfileId tenantProfileId = tenantIds.get(tenantId); - if (tenantProfileId != null) { - profile = profiles.get(tenantProfileId); - } + TenantProfile profile = lookupCached(tenantId); if (profile == null) { - tenantProfileFetchLock.lock(); + // Per-tenant lock: de-duplicates concurrent misses for the SAME tenant while allowing + // different tenants to resolve their profiles concurrently. A single global lock here + // serializes the synchronous cross-service fetch below across the entire process. + Lock lock = tenantProfileFetchLocks.get(tenantId); + lock.lock(); try { - tenantProfileId = tenantIds.get(tenantId); - if (tenantProfileId != null) { - profile = profiles.get(tenantProfileId); - } + profile = lookupCached(tenantId); if (profile == null) { - TransportProtos.GetEntityProfileRequestMsg msg = TransportProtos.GetEntityProfileRequestMsg.newBuilder() - .setEntityType(EntityType.TENANT.name()) - .setEntityIdMSB(tenantId.getId().getMostSignificantBits()) - .setEntityIdLSB(tenantId.getId().getLeastSignificantBits()) - .build(); - TransportProtos.GetEntityProfileResponseMsg entityProfileMsg = transportService.getEntityProfile(msg); - profile = ProtoUtils.fromProto(entityProfileMsg.getTenantProfile()); - TenantProfile existingProfile = profiles.get(profile.getId()); - if (existingProfile != null) { - profile = existingProfile; - } else { - profiles.put(profile.getId(), profile); - } - tenantProfileIds.computeIfAbsent(profile.getId(), id -> ConcurrentHashMap.newKeySet()).add(tenantId); - tenantIds.put(tenantId, profile.getId()); - ApiUsageState apiUsageState = ProtoUtils.fromProto(entityProfileMsg.getApiState()); - rateLimitService.update(tenantId, apiUsageState.isTransportEnabled()); + profile = fetchAndCacheTenantProfile(tenantId); } } finally { - tenantProfileFetchLock.unlock(); + lock.unlock(); } } return profile; } + private TenantProfile lookupCached(TenantId tenantId) { + TenantProfileId tenantProfileId = tenantIds.get(tenantId); + if (tenantProfileId != null) { + return profiles.get(tenantProfileId); + } + return null; + } + + private TenantProfile fetchAndCacheTenantProfile(TenantId tenantId) { + TransportProtos.GetEntityProfileRequestMsg msg = TransportProtos.GetEntityProfileRequestMsg.newBuilder() + .setEntityType(EntityType.TENANT.name()) + .setEntityIdMSB(tenantId.getId().getMostSignificantBits()) + .setEntityIdLSB(tenantId.getId().getLeastSignificantBits()) + .build(); + TransportProtos.GetEntityProfileResponseMsg entityProfileMsg = transportService.getEntityProfile(msg); + TenantProfile profile = ProtoUtils.fromProto(entityProfileMsg.getTenantProfile()); + TenantProfile existingProfile = profiles.get(profile.getId()); + if (existingProfile != null) { + profile = existingProfile; + } else { + profiles.put(profile.getId(), profile); + } + tenantProfileIds.computeIfAbsent(profile.getId(), id -> ConcurrentHashMap.newKeySet()).add(tenantId); + tenantIds.put(tenantId, profile.getId()); + ApiUsageState apiUsageState = ProtoUtils.fromProto(entityProfileMsg.getApiState()); + rateLimitService.update(tenantId, apiUsageState.isTransportEnabled()); + return profile; + } + } diff --git a/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitServiceTest.java b/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitServiceTest.java new file mode 100644 index 0000000000..0ca00ed1e4 --- /dev/null +++ b/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/limits/DefaultTransportRateLimitServiceTest.java @@ -0,0 +1,94 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.limits; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.transport.TransportTenantProfileCache; + +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DefaultTransportRateLimitServiceTest { + + private TransportTenantProfileCache tenantProfileCache; + private ExecutorService executor; + + private final TenantId tenant = TenantId.fromUUID(UUID.randomUUID()); + + @BeforeEach + void setUp() { + tenantProfileCache = mock(TransportTenantProfileCache.class); + executor = Executors.newCachedThreadPool(); + } + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void checkLimitsDoesNotHoldMapBinLockAcrossProfileFetch() throws Exception { + // Two concurrent rate-limit checks for the SAME tenant must both be able to reach + // the (blocking) tenant-profile fetch concurrently. If the blocking fetch runs inside + // ConcurrentHashMap.computeIfAbsent, the second caller is stuck on the bin reservation + // node and never reaches the fetch -> the latch never reaches zero. + CountDownLatch bothCallersReachedFetch = new CountDownLatch(2); + CountDownLatch releaseFetch = new CountDownLatch(1); + + when(tenantProfileCache.get(tenant)).thenAnswer(invocation -> { + bothCallersReachedFetch.countDown(); + releaseFetch.await(5, TimeUnit.SECONDS); + return tenantProfile(); + }); + + DefaultTransportRateLimitService service = new DefaultTransportRateLimitService(tenantProfileCache); + + Runnable check = () -> service.checkLimits(tenant, null, null, 1, false); + executor.submit(check); + executor.submit(check); + + boolean bothReached = bothCallersReachedFetch.await(3, TimeUnit.SECONDS); + releaseFetch.countDown(); + + assertThat(bothReached) + .as("both checkLimits calls should reach the profile fetch concurrently (no bin lock across I/O)") + .isTrue(); + } + + private TenantProfile tenantProfile() { + TenantProfile profile = new TenantProfile(new TenantProfileId(UUID.randomUUID())); + profile.setName("test-profile"); + TenantProfileData profileData = new TenantProfileData(); + profileData.setConfiguration(new DefaultTenantProfileConfiguration()); + profile.setProfileData(profileData); + return profile; + } + +} diff --git a/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCacheTest.java b/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCacheTest.java new file mode 100644 index 0000000000..4da7767330 --- /dev/null +++ b/common/transport/transport-api/src/test/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCacheTest.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport.service; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.ApiUsageStateId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.limits.TransportRateLimitService; +import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; + +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DefaultTransportTenantProfileCacheTest { + + private DefaultTransportTenantProfileCache cache; + private TransportService transportService; + private TransportRateLimitService rateLimitService; + private ExecutorService executor; + + private final TenantId tenantA = TenantId.fromUUID(UUID.randomUUID()); + private final TenantId tenantB = TenantId.fromUUID(UUID.randomUUID()); + + @BeforeEach + void setUp() { + cache = new DefaultTransportTenantProfileCache(); + transportService = mock(TransportService.class); + rateLimitService = mock(TransportRateLimitService.class); + doNothing().when(rateLimitService).update(any(TenantId.class), anyBoolean()); + cache.setTransportService(transportService); + cache.setRateLimitService(rateLimitService); + executor = Executors.newCachedThreadPool(); + } + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void fetchForOneTenantDoesNotBlockResolutionOfAnotherTenant() throws Exception { + CountDownLatch tenantAFetchStarted = new CountDownLatch(1); + CountDownLatch releaseTenantA = new CountDownLatch(1); + + GetEntityProfileResponseMsg responseA = responseFor(tenantA); + GetEntityProfileResponseMsg responseB = responseFor(tenantB); + + when(transportService.getEntityProfile(any())).thenAnswer(invocation -> { + GetEntityProfileRequestMsg msg = invocation.getArgument(0); + TenantId requested = TenantId.fromUUID(new UUID(msg.getEntityIdMSB(), msg.getEntityIdLSB())); + if (requested.equals(tenantA)) { + tenantAFetchStarted.countDown(); + releaseTenantA.await(5, TimeUnit.SECONDS); + return responseA; + } + return responseB; + }); + + // T1 starts fetching tenantA's profile and blocks inside the cross-service round-trip. + Future tenantAResult = executor.submit(() -> cache.get(tenantA)); + assertThat(tenantAFetchStarted.await(5, TimeUnit.SECONDS)) + .as("tenantA fetch should have started").isTrue(); + + // T2 resolves a different tenant - it must NOT wait for tenantA's in-flight fetch. + // Fails today (single global lock); passes once locking is per-tenant. + TenantProfile tenantBProfile = CompletableFuture + .supplyAsync(() -> cache.get(tenantB), executor) + .get(2, TimeUnit.SECONDS); + assertThat(tenantBProfile).isNotNull(); + + releaseTenantA.countDown(); + assertThat(tenantAResult.get(5, TimeUnit.SECONDS)).isNotNull(); + } + + private GetEntityProfileResponseMsg responseFor(TenantId tenantId) { + TenantProfile profile = new TenantProfile(new TenantProfileId(UUID.randomUUID())); + profile.setName("profile-" + tenantId.getId()); + return GetEntityProfileResponseMsg.newBuilder() + .setEntityType(EntityType.TENANT.name()) + .setTenantProfile(ProtoUtils.toProto(profile)) + .setApiState(ProtoUtils.toProto(enabledApiUsageState(tenantId))) + .build(); + } + + private ApiUsageState enabledApiUsageState(TenantId tenantId) { + ApiUsageState state = new ApiUsageState(new ApiUsageStateId(UUID.randomUUID())); + state.setTenantId(tenantId); + state.setEntityId(tenantId); + state.setTransportState(ApiUsageStateValue.ENABLED); + state.setDbStorageState(ApiUsageStateValue.ENABLED); + state.setReExecState(ApiUsageStateValue.ENABLED); + state.setJsExecState(ApiUsageStateValue.ENABLED); + state.setTbelExecState(ApiUsageStateValue.ENABLED); + state.setEmailExecState(ApiUsageStateValue.ENABLED); + state.setSmsExecState(ApiUsageStateValue.ENABLED); + state.setAlarmExecState(ApiUsageStateValue.ENABLED); + state.setVersion(1L); + return state; + } + +}